diff --git a/wandb/run-20240802_173428-s75vpwte/files/config.yaml b/wandb/run-20240802_173428-s75vpwte/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e7f48280da6a19c9fd5e8ec47aa87d71f2479e7 --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 512 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3 +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-mistral-sample_train_2024-08-02-17:34:15 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-mistral-sample +base_model: + desc: null + value: /share/pretrained_lm/custom/tiny-mistral +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-mistral-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32768 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722587668.341658 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 256 +model_type: + desc: null + value: mistral +max_position_embeddings: + desc: null + value: 512 +num_attention_heads: + desc: null + value: 4 +num_hidden_layers: + desc: null + value: 4 +model_architecture: + desc: null + value: MistralForCausalLM diff --git a/wandb/run-20240802_173428-s75vpwte/files/output.log b/wandb/run-20240802_173428-s75vpwte/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..b4b291c8a6b95ecd4f1fa8c66378089368e7b60c --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/files/output.log @@ -0,0 +1,4413 @@ +Created Hugging Face repository with ID koichi12/tiny-mistral-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample, skipping model loading +--> Model /share/pretrained_lm/custom/tiny-mistral +--> /share/pretrained_lm/custom/tiny-mistral has 19.925248 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 323200 + test: 3200 +> building train, validation, and test datasets for GPT ... +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-mistral-sample, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-mistral-sample/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): MistralForCausalLM( + (model): MistralModel( + (embed_tokens): Embedding(32768, 256) + (layers): ModuleList( + (0-3): 4 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): MistralDecoderLayer( + (self_attn): MistralFlashAttention2( + (q_proj): Linear(in_features=256, out_features=512, bias=False) + (k_proj): Linear(in_features=256, out_features=256, bias=False) + (v_proj): Linear(in_features=256, out_features=256, bias=False) + (o_proj): Linear(in_features=512, out_features=256, bias=False) + (rotary_emb): MistralRotaryEmbedding() + ) + (mlp): MistralMLP( + (gate_proj): Linear(in_features=256, out_features=512, bias=False) + (up_proj): Linear(in_features=256, out_features=512, bias=False) + (down_proj): Linear(in_features=512, out_features=256, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): MistralRMSNorm() + (post_attention_layernorm): MistralRMSNorm() + ) + ) + ) + ) + (norm): MistralRMSNorm() + ) + (lm_head): Linear(in_features=256, out_features=32768, bias=False) + ) +) +model config: MistralConfig { + "_name_or_path": "/share/pretrained_lm/custom/tiny-mistral", + "architectures": [ + "MistralForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 256, + "initializer_range": 0.02, + "intermediate_size": 512, + "label_smoothing": 0.0, + "max_position_embeddings": 512, + "model_type": "mistral", + "num_attention_heads": 4, + "num_hidden_layers": 4, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-05, + "rope_theta": 1000000.0, + "sliding_window": 4096, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32768 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 4.191714076411825, Tokens per sec: 59212.387573712935, Loss: 10.457756042480469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 10.155863374650547, Tokens per sec: 143462.29426035954, Loss: 10.457653999328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 10.20466875804562, Tokens per sec: 144151.72183690514, Loss: 10.457398414611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 10.218813081470659, Tokens per sec: 144351.52534099534, Loss: 10.45959758758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 10.16693731005277, Tokens per sec: 143618.72529144835, Loss: 10.455076217651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 10.25290803648562, Tokens per sec: 144833.15258318128, Loss: 10.458924293518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 10.296831519740095, Tokens per sec: 145453.61816519394, Loss: 10.456448554992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 10.298754005439802, Tokens per sec: 145480.77530575288, Loss: 10.453125953674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 10.242243008531943, Tokens per sec: 144682.49780158937, Loss: 10.458549499511719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 10.220693139054346, Tokens per sec: 144378.08313961347, Loss: 10.457084655761719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 10.225627061819914, Tokens per sec: 144447.7800086575, Loss: 10.455498695373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 10.1606166726306, Tokens per sec: 143529.4396135772, Loss: 10.459282875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 10.24396633723883, Tokens per sec: 144706.84163932467, Loss: 10.458690643310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 10.24877545782651, Tokens per sec: 144774.77554582106, Loss: 10.458513259887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 10.232591085965593, Tokens per sec: 144546.15420338302, Loss: 10.458674430847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 10.222365081023877, Tokens per sec: 144401.7010854218, Loss: 10.45798397064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 9.482495375654874, Tokens per sec: 133950.2602309779, Loss: 10.453550338745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 10.223809895149355, Tokens per sec: 144422.110610597, Loss: 10.460649490356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 10.237859845439331, Tokens per sec: 144620.58099450107, Loss: 10.455191612243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 10.238595141048929, Tokens per sec: 144630.9678214227, Loss: 10.454816818237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 10.272531921205129, Tokens per sec: 145110.36067670374, Loss: 10.45626449584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 10.247119189977502, Tokens per sec: 144751.37901351624, Loss: 10.457989692687988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 10.163370911865934, Tokens per sec: 143568.34615111782, Loss: 10.459192276000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 10.279620233416827, Tokens per sec: 145210.49057160388, Loss: 10.45911693572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 10.221122420655343, Tokens per sec: 144384.14719552788, Loss: 10.454962730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 10.198525660529715, Tokens per sec: 144064.94409768342, Loss: 10.456686973571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 10.224930558160116, Tokens per sec: 144437.94115898918, Loss: 10.459595680236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 10.092682885877377, Tokens per sec: 142569.80314094233, Loss: 10.45623779296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 10.129997078447364, Tokens per sec: 143096.90551294992, Loss: 10.455552101135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 10.022802778558065, Tokens per sec: 141582.67283509247, Loss: 10.45615291595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 10.139056339500133, Tokens per sec: 143224.87714145545, Loss: 10.457874298095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 9.488115804897697, Tokens per sec: 134029.65472893027, Loss: 10.45525074005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 10.197773691950768, Tokens per sec: 144054.32174746387, Loss: 10.45787525177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 10.150694496466382, Tokens per sec: 143389.27839792645, Loss: 10.454483985900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 10.163369498591285, Tokens per sec: 143568.326187121, Loss: 10.456534385681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 10.146694953169394, Tokens per sec: 143332.78062553503, Loss: 10.457503318786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 10.153948462856501, Tokens per sec: 143435.24410921574, Loss: 10.451608657836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 10.177799156913375, Tokens per sec: 143772.16034793173, Loss: 10.453251838684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 9.77233166514733, Tokens per sec: 138044.50387295955, Loss: 10.454899787902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 10.179752466999773, Tokens per sec: 143799.75291550174, Loss: 10.453554153442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 10.150187809598693, Tokens per sec: 143382.12091088385, Loss: 10.45371150970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 10.113596066997061, Tokens per sec: 142865.223907551, Loss: 10.454233169555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 10.217986602792648, Tokens per sec: 144339.8504569475, Loss: 10.454697608947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 10.196508758423741, Tokens per sec: 144036.45322568688, Loss: 10.451160430908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 10.23397073007084, Tokens per sec: 144565.64313320612, Loss: 10.454113006591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 9.414973420450213, Tokens per sec: 132996.44131384033, Loss: 10.450791358947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 10.094134795103635, Tokens per sec: 142590.31289190814, Loss: 10.453559875488281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 10.086694437614081, Tokens per sec: 142485.20998571574, Loss: 10.450769424438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 10.080042763585883, Tokens per sec: 142391.24806622605, Loss: 10.45075798034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 9.653714869237353, Tokens per sec: 136368.91637721463, Loss: 10.450355529785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 10.13985679193107, Tokens per sec: 143236.18437728094, Loss: 10.451664924621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 10.159771716804602, Tokens per sec: 143517.50372030312, Loss: 10.45103645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 10.148574267015356, Tokens per sec: 143359.3279180724, Loss: 10.450456619262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 10.12846183940091, Tokens per sec: 143075.21864028167, Loss: 10.45256233215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 10.129292457118067, Tokens per sec: 143086.95199262805, Loss: 10.4503812789917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 10.129821333345943, Tokens per sec: 143094.42292781416, Loss: 10.450446128845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 10.13418094591192, Tokens per sec: 143156.00705884554, Loss: 10.447710990905762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 9.844077576111138, Tokens per sec: 139057.99062548508, Loss: 10.451488494873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 10.073754600134757, Tokens per sec: 142302.4211174868, Loss: 10.451848983764648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 10.09874691763735, Tokens per sec: 142655.46399287196, Loss: 10.448508262634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 9.384858662327444, Tokens per sec: 132571.03855564925, Loss: 10.44882583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 10.115082490408712, Tokens per sec: 142886.2212078308, Loss: 10.446773529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 10.128209827722918, Tokens per sec: 143071.65870921803, Loss: 10.451287269592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 10.171707036196164, Tokens per sec: 143686.10270982058, Loss: 10.449091911315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 10.088559331415002, Tokens per sec: 142511.5535798901, Loss: 10.449789047241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 10.157084854614704, Tokens per sec: 143479.54895467617, Loss: 10.445674896240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 10.121424248362183, Tokens per sec: 142975.8052355088, Loss: 10.448121070861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 10.146376761191528, Tokens per sec: 143328.28582785252, Loss: 10.446863174438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 10.156654392955154, Tokens per sec: 143473.46822918864, Loss: 10.44879150390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 10.166077615972004, Tokens per sec: 143606.5812047628, Loss: 10.449258804321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 10.130323710512409, Tokens per sec: 143101.51953577614, Loss: 10.447320938110352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 10.096720220158995, Tokens per sec: 142626.8347508971, Loss: 10.446844100952148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 10.113716228744064, Tokens per sec: 142866.92131911236, Loss: 10.447033882141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 10.102502898026055, Tokens per sec: 142708.52118199342, Loss: 10.445926666259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 10.144630103943758, Tokens per sec: 143303.61244984344, Loss: 10.447366714477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 9.376901817124772, Tokens per sec: 132458.63971512337, Loss: 10.446215629577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 10.102494537216836, Tokens per sec: 142708.4030767346, Loss: 10.446431159973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 10.116675465543183, Tokens per sec: 142908.72366370878, Loss: 10.4468355178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 10.168741308013791, Tokens per sec: 143644.20866758112, Loss: 10.44129467010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 10.200766329394186, Tokens per sec: 144096.59591143046, Loss: 10.44300365447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 10.170385507066854, Tokens per sec: 143667.43471539917, Loss: 10.444250106811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 10.172411284570162, Tokens per sec: 143696.050961755, Loss: 10.44077205657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 10.108668867846049, Tokens per sec: 142795.6220166624, Loss: 10.440753936767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 10.099261225438262, Tokens per sec: 142662.72913364365, Loss: 10.440824508666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 10.174923327405212, Tokens per sec: 143731.5362193941, Loss: 10.441353797912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 10.174703072691694, Tokens per sec: 143728.4248889875, Loss: 10.441535949707031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 10.148704593971871, Tokens per sec: 143361.1689239521, Loss: 10.439326286315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 10.185786549848876, Tokens per sec: 143884.99070744074, Loss: 10.440052032470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 10.134744661423913, Tokens per sec: 143163.97013570834, Loss: 10.436569213867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 9.363371165711962, Tokens per sec: 132267.50497621344, Loss: 10.438343048095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 10.176540973649313, Tokens per sec: 143754.38718074706, Loss: 10.436022758483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 10.178465442297435, Tokens per sec: 143781.57233254626, Loss: 10.43701171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 10.251993531889303, Tokens per sec: 144820.2342400863, Loss: 10.434489250183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 10.169976781547856, Tokens per sec: 143661.66103584922, Loss: 10.434334754943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 10.070538381620114, Tokens per sec: 142256.98863479853, Loss: 10.433130264282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 10.129717847648239, Tokens per sec: 143092.96108305827, Loss: 10.434069633483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 10.160114750802496, Tokens per sec: 143522.34943775044, Loss: 10.431957244873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 10.164635860287605, Tokens per sec: 143586.2148832974, Loss: 10.433119773864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 10.127938966009761, Tokens per sec: 143067.83250150306, Loss: 10.433173179626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 10.134068374705699, Tokens per sec: 143154.41687168798, Loss: 10.429265022277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 10.164240131701257, Tokens per sec: 143580.62479914527, Loss: 10.42747974395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 10.077360324245099, Tokens per sec: 142353.35577801315, Loss: 10.432609558105469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 10.090668320683987, Tokens per sec: 142541.3452803037, Loss: 10.429868698120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 10.0640609840616, Tokens per sec: 142165.4885544705, Loss: 10.42664909362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 9.317291574797053, Tokens per sec: 131616.5820967533, Loss: 10.42693042755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 10.006773775547307, Tokens per sec: 141356.24624172476, Loss: 10.426505088806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 10.069759445742323, Tokens per sec: 142245.98534300667, Loss: 10.42402458190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 10.085263590534067, Tokens per sec: 142464.99775980625, Loss: 10.425642967224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 10.048620876682506, Tokens per sec: 141947.380733745, Loss: 10.42268180847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 10.10012253317142, Tokens per sec: 142674.8960148732, Loss: 10.421835899353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 10.071783320071486, Tokens per sec: 142274.57470501808, Loss: 10.419801712036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 10.081455591179571, Tokens per sec: 142411.2057478635, Loss: 10.424200057983398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 10.163625351927818, Tokens per sec: 143571.9403856681, Loss: 10.419133186340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 10.210003502181042, Tokens per sec: 144227.08073104604, Loss: 10.420833587646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 10.266824446770123, Tokens per sec: 145029.7365734963, Loss: 10.422022819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 10.276326145800736, Tokens per sec: 145163.95810563173, Loss: 10.418856620788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 10.278296106961296, Tokens per sec: 145191.78588720693, Loss: 10.416363716125488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 10.273932857263956, Tokens per sec: 145130.1503778544, Loss: 10.41781997680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 9.504005589915723, Tokens per sec: 134254.1147211432, Loss: 10.415939331054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 10.255977947153061, Tokens per sec: 144876.51831303394, Loss: 10.414381980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 9.81994741640713, Tokens per sec: 138717.12663940128, Loss: 10.41608715057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 10.269675117598187, Tokens per sec: 145070.00530911123, Loss: 10.410514831542969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 9.949922430108606, Tokens per sec: 140553.16095517177, Loss: 10.41154956817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 10.231399868328593, Tokens per sec: 144529.326996393, Loss: 10.406919479370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 10.232656706218837, Tokens per sec: 144547.08115875185, Loss: 10.411781311035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 10.26497060384771, Tokens per sec: 145003.54908465006, Loss: 10.411126136779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 10.217435472678329, Tokens per sec: 144332.0651621164, Loss: 10.408699035644531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 10.220248101083072, Tokens per sec: 144371.796508331, Loss: 10.408679962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 10.277805070682318, Tokens per sec: 145184.84948125615, Loss: 10.407244682312012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 10.267377752094285, Tokens per sec: 145037.55259546338, Loss: 10.407532691955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 10.333425300186265, Tokens per sec: 145970.54395523274, Loss: 10.407878875732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 10.353269256141663, Tokens per sec: 146250.86078734658, Loss: 10.405104637145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 10.300370686656361, Tokens per sec: 145503.6126350728, Loss: 10.405129432678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 9.56055043893834, Tokens per sec: 135052.87042217384, Loss: 10.40306282043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 10.266469915698716, Tokens per sec: 145024.7284477453, Loss: 10.404004096984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 10.347329889993375, Tokens per sec: 146166.9609688228, Loss: 10.401853561401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 10.291670716572726, Tokens per sec: 145380.71637089993, Loss: 10.401577949523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 10.263169583882176, Tokens per sec: 144978.1077758482, Loss: 10.39961051940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 10.279093102226325, Tokens per sec: 145203.04428691335, Loss: 10.398153305053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 10.271244407650343, Tokens per sec: 145092.17318819126, Loss: 10.39930534362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 10.271348935095935, Tokens per sec: 145093.6497487361, Loss: 10.398695945739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 10.246768304336298, Tokens per sec: 144746.4223833162, Loss: 10.392829895019531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 10.258438291208035, Tokens per sec: 144911.27327081305, Loss: 10.395927429199219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 10.223362876905476, Tokens per sec: 144415.79600587286, Loss: 10.397538185119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 10.280867109198805, Tokens per sec: 145228.104008664, Loss: 10.395589828491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 10.264773679008009, Tokens per sec: 145000.76731334632, Loss: 10.391180992126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 10.290648416765883, Tokens per sec: 145366.27530662983, Loss: 10.391834259033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 10.316656550072162, Tokens per sec: 145733.66765289367, Loss: 10.391467094421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 9.54896490937807, Tokens per sec: 134889.21258338424, Loss: 10.392541885375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 10.369769353324173, Tokens per sec: 146483.94208334255, Loss: 10.386697769165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 10.371900247012409, Tokens per sec: 146514.04320680807, Loss: 10.391349792480469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 10.386463412700646, Tokens per sec: 146719.76330014283, Loss: 10.386529922485352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 10.377267111334099, Tokens per sec: 146589.85583249733, Loss: 10.389164924621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 10.382109852705657, Tokens per sec: 146658.26466806786, Loss: 10.391218185424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 10.38171036524981, Tokens per sec: 146652.6214859149, Loss: 10.383843421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 10.375397035169476, Tokens per sec: 146563.43903196338, Loss: 10.38492202758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 10.378074747535443, Tokens per sec: 146601.26454666554, Loss: 10.38614559173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 10.308530211189327, Tokens per sec: 145618.87453515848, Loss: 10.38485050201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 10.2506671723997, Tokens per sec: 144801.49801172514, Loss: 10.38524341583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 10.282530466005882, Tokens per sec: 145251.60067998714, Loss: 10.382554054260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 10.288649761562876, Tokens per sec: 145338.0421914055, Loss: 10.382027626037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 10.303519434416868, Tokens per sec: 145548.0920221131, Loss: 10.379608154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 9.529968467421954, Tokens per sec: 134620.86778144352, Loss: 10.37621784210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 10.276700078867616, Tokens per sec: 145169.24030505636, Loss: 10.380071640014648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 10.277425394670855, Tokens per sec: 145179.48615667503, Loss: 10.378073692321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 9.859104945108697, Tokens per sec: 139270.26808073994, Loss: 10.378182411193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 10.293807160830175, Tokens per sec: 145410.89590201667, Loss: 10.376781463623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 10.290226214039414, Tokens per sec: 145360.31124729308, Loss: 10.374388694763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 10.2975080888964, Tokens per sec: 145463.17541895056, Loss: 10.376664161682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 10.303223165554483, Tokens per sec: 145543.90691158653, Loss: 10.37369441986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 10.312143044889089, Tokens per sec: 145669.90962614276, Loss: 10.372196197509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 10.308708243251612, Tokens per sec: 145621.38942603138, Loss: 10.370882034301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 10.304333877906629, Tokens per sec: 145559.59689641834, Loss: 10.369919776916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 10.310735566722746, Tokens per sec: 145650.02751081527, Loss: 10.371394157409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 10.189642240767295, Tokens per sec: 143939.45641308383, Loss: 10.367630004882812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 10.200084452973085, Tokens per sec: 144086.96368695438, Loss: 10.36749267578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 10.272600070625696, Tokens per sec: 145111.3233592317, Loss: 10.364474296569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 9.499224436069525, Tokens per sec: 134186.5758744018, Loss: 10.363327026367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 10.252121608424275, Tokens per sec: 144822.04345638532, Loss: 10.36156940460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 10.19886408436491, Tokens per sec: 144069.7246917145, Loss: 10.357871055603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 10.183734568486633, Tokens per sec: 143856.00430390742, Loss: 10.356569290161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 10.14103799972132, Tokens per sec: 143252.8701846157, Loss: 10.35228157043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 10.209169730151768, Tokens per sec: 144215.30282071017, Loss: 10.355916976928711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 10.220510358777313, Tokens per sec: 144375.50117519338, Loss: 10.350372314453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 10.206524260558524, Tokens per sec: 144177.93276921954, Loss: 10.346668243408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 10.221111668847051, Tokens per sec: 144383.99531488237, Loss: 10.340231895446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 10.215047480084401, Tokens per sec: 144298.33224512413, Loss: 10.34239673614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 10.199217971668796, Tokens per sec: 144074.7237235695, Loss: 10.340786933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 10.19669440313215, Tokens per sec: 144039.07565322486, Loss: 10.33640193939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 10.20085758982118, Tokens per sec: 144097.88506132827, Loss: 10.330053329467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 10.17370269256226, Tokens per sec: 143714.2934633069, Loss: 10.329814910888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 9.354935623950004, Tokens per sec: 132148.3440413076, Loss: 10.326383590698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 10.072845939265822, Tokens per sec: 142289.5853232118, Loss: 10.321505546569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 10.083499262842132, Tokens per sec: 142440.0747681142, Loss: 10.321043014526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 10.175095919185681, Tokens per sec: 143733.97426054164, Loss: 10.31937313079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 10.220433226996974, Tokens per sec: 144374.41160734874, Loss: 10.316153526306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 10.210891124446048, Tokens per sec: 144239.61933282478, Loss: 10.314833641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 10.20722655176236, Tokens per sec: 144187.8533740588, Loss: 10.309176445007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 10.198955078026119, Tokens per sec: 144071.01007326393, Loss: 10.304411888122559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 10.21604702711165, Tokens per sec: 144312.45190235664, Loss: 10.30544376373291 +------------------------------------------------------------------ +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +[rank0]:[2024-08-02 17:38:25,573] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0014257580000958114, 'preprocessing_with_comm': 0.0006898800002090866, 'state_converting': 0.027989020999939385, : 0.030476782999812713}) + eval ppl=29907.634765625, eval loss=10.305869102478027 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000200, took 0.17s +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 10.14542675128447, Tokens per sec: 143314.86593475146, Loss: 10.303635597229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 10.218009602649772, Tokens per sec: 144340.1753542161, Loss: 10.297120094299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 10.161242798602416, Tokens per sec: 143538.28430408743, Loss: 10.297996520996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 10.278034293281461, Tokens per sec: 145188.08749251688, Loss: 10.295552253723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 10.258902247457705, Tokens per sec: 144917.82714275466, Loss: 10.29259204864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 10.299596851689186, Tokens per sec: 145492.6813990297, Loss: 10.286720275878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 9.186382445872306, Tokens per sec: 129767.3524170742, Loss: 10.287761688232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 10.1765980258049, Tokens per sec: 143755.19310268905, Loss: 10.283676147460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 10.295652945927543, Tokens per sec: 145436.9695655755, Loss: 10.27834701538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 10.281767796496831, Tokens per sec: 145240.8271678302, Loss: 10.279411315917969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 10.225471952118868, Tokens per sec: 144445.58892034198, Loss: 10.279814720153809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 9.860548304147091, Tokens per sec: 139290.65705127356, Loss: 10.271328926086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 10.258731084447946, Tokens per sec: 144915.40928450206, Loss: 10.265256881713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 10.265459398879521, Tokens per sec: 145010.453830618, Loss: 10.265909194946289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 10.25108901067547, Tokens per sec: 144807.4569228109, Loss: 10.266912460327148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 10.26265880240946, Tokens per sec: 144970.8924481859, Loss: 10.261746406555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 10.178260857560076, Tokens per sec: 143778.68235709963, Loss: 10.257867813110352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 10.315488157568508, Tokens per sec: 145717.16287501442, Loss: 10.255387306213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 10.375179251648087, Tokens per sec: 146560.36260975502, Loss: 10.253173828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 10.288415283227955, Tokens per sec: 145334.72993732712, Loss: 10.254060745239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 10.234796416409564, Tokens per sec: 144577.30682462486, Loss: 10.25122356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 9.480164448528672, Tokens per sec: 133917.3334239756, Loss: 10.247918128967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 10.251983890778797, Tokens per sec: 144820.09804921987, Loss: 10.24617862701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 10.2683737939195, Tokens per sec: 145051.62273801584, Loss: 10.240598678588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 10.27285362527574, Tokens per sec: 145114.90508640485, Loss: 10.23851203918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 9.736625032303477, Tokens per sec: 137540.1099795878, Loss: 10.234407424926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 10.215635314828958, Tokens per sec: 144306.63603161566, Loss: 10.23804759979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 10.303028003912193, Tokens per sec: 145541.15004730807, Loss: 10.232616424560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 10.227216039786141, Tokens per sec: 144470.2260003132, Loss: 10.231459617614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 10.158113144384778, Tokens per sec: 143494.07463350202, Loss: 10.229045867919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 10.173033682112846, Tokens per sec: 143704.84298426672, Loss: 10.223368644714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 10.16633622932275, Tokens per sec: 143610.23439142507, Loss: 10.224311828613281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 10.320637556300797, Tokens per sec: 145789.9035696204, Loss: 10.216556549072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 10.209337763420447, Tokens per sec: 144217.67646806512, Loss: 10.214221954345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 9.833222008979629, Tokens per sec: 138904.64427680627, Loss: 10.218131065368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 10.207503051811566, Tokens per sec: 144191.7592292243, Loss: 10.21284294128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 9.077969244203945, Tokens per sec: 128235.90146448702, Loss: 10.213790893554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 10.137813093290708, Tokens per sec: 143207.31497594036, Loss: 10.209451675415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 10.181763860092282, Tokens per sec: 143828.16596686587, Loss: 10.205364227294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 10.10604041755567, Tokens per sec: 142758.4923807963, Loss: 10.20529842376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 10.127328675489986, Tokens per sec: 143059.21150347436, Loss: 10.197296142578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 10.126761345868395, Tokens per sec: 143051.19737349715, Loss: 10.195286750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 10.2273772509538, Tokens per sec: 144472.50327828748, Loss: 10.196257591247559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 10.198139397368468, Tokens per sec: 144059.48772265585, Loss: 10.196325302124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 10.251738491753354, Tokens per sec: 144816.63152885617, Loss: 10.191588401794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 10.267478173093519, Tokens per sec: 145038.9711481172, Loss: 10.184664726257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 10.153094565336575, Tokens per sec: 143423.18190507288, Loss: 10.186479568481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 10.2323565339997, Tokens per sec: 144542.8409091894, Loss: 10.184415817260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 10.217464264336153, Tokens per sec: 144332.47187468575, Loss: 10.189361572265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 10.162508808087363, Tokens per sec: 143556.16802490622, Loss: 10.178735733032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 10.180452960460673, Tokens per sec: 143809.6481253237, Loss: 10.179248809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 9.417614844675242, Tokens per sec: 133033.75422023327, Loss: 10.17383861541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 10.133486725973563, Tokens per sec: 143146.20046915408, Loss: 10.173090934753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 10.135737367846158, Tokens per sec: 143177.99316217186, Loss: 10.166394233703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 10.171021721713368, Tokens per sec: 143676.42191909262, Loss: 10.159197807312012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 10.213226962323837, Tokens per sec: 144272.61550937875, Loss: 10.16612434387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 10.140536665318173, Tokens per sec: 143245.7883067867, Loss: 10.161408424377441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 10.147993291463774, Tokens per sec: 143351.12102492465, Loss: 10.161805152893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 10.081369743032182, Tokens per sec: 142409.99305213022, Loss: 10.15499210357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 10.136414699599161, Tokens per sec: 143187.56118841213, Loss: 10.156997680664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 10.13894938322049, Tokens per sec: 143223.3662710649, Loss: 10.152349472045898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 10.022590331066533, Tokens per sec: 141579.67178994042, Loss: 10.151415824890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 10.108760404741748, Tokens per sec: 142796.91507197265, Loss: 10.153284072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 10.120016306691703, Tokens per sec: 142955.91657269595, Loss: 10.146202087402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 10.112395803315263, Tokens per sec: 142848.26891562607, Loss: 10.15228271484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 10.07233849996177, Tokens per sec: 142282.41720721108, Loss: 10.143117904663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 9.42909119037458, Tokens per sec: 133195.86972169438, Loss: 10.135619163513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 10.236926236534627, Tokens per sec: 144607.3927828769, Loss: 10.139607429504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 10.226744829366195, Tokens per sec: 144463.56965555644, Loss: 10.137415885925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 10.235068120133121, Tokens per sec: 144581.14492662592, Loss: 10.137939453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 10.217314100340309, Tokens per sec: 144330.3506496786, Loss: 10.135587692260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 10.179005225670286, Tokens per sec: 143789.19734267253, Loss: 10.132683753967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 10.137080051738725, Tokens per sec: 143196.95998996266, Loss: 10.128678321838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 10.17038637933888, Tokens per sec: 143667.4470371626, Loss: 10.126481056213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 10.098386427888821, Tokens per sec: 142650.3716945145, Loss: 10.12547492980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 10.064654206595042, Tokens per sec: 142173.8684491693, Loss: 10.122909545898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 10.18138579642544, Tokens per sec: 143822.825418355, Loss: 10.122684478759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 10.072890344710297, Tokens per sec: 142290.212597005, Loss: 10.12460994720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 10.127434910143538, Tokens per sec: 143060.71218013437, Loss: 10.121485710144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 10.146650984309632, Tokens per sec: 143332.1595189619, Loss: 10.11555004119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 9.366087111085436, Tokens per sec: 132305.87057251882, Loss: 10.112740516662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 10.246505953175266, Tokens per sec: 144742.71639613668, Loss: 10.10966968536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 9.853335615024312, Tokens per sec: 139188.7702011685, Loss: 10.107421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 10.24674187707493, Tokens per sec: 144746.0490703435, Loss: 10.111739158630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 10.274680629385417, Tokens per sec: 145140.71344868073, Loss: 10.106405258178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 10.268261487999812, Tokens per sec: 145050.0362983107, Loss: 10.10575008392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 10.278385178105317, Tokens per sec: 145193.044111171, Loss: 10.099571228027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 10.273069831318168, Tokens per sec: 145117.95922505713, Loss: 10.098198890686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 10.290662194931837, Tokens per sec: 145366.46993777298, Loss: 10.102943420410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 10.313752866158366, Tokens per sec: 145692.65005146357, Loss: 10.093036651611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 10.265339421895296, Tokens per sec: 145008.75902902597, Loss: 10.090638160705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 10.296925419509002, Tokens per sec: 145454.94459858327, Loss: 10.094037055969238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 10.329221136853874, Tokens per sec: 145911.1557087725, Loss: 10.092416763305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 10.256767691818652, Tokens per sec: 144887.67429036702, Loss: 10.088025093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 10.258503727217354, Tokens per sec: 144912.1976235419, Loss: 10.084918975830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 9.421271723186637, Tokens per sec: 133085.41149069098, Loss: 10.082039833068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 10.278397950729245, Tokens per sec: 145193.22453797123, Loss: 10.087653160095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 10.218611885433674, Tokens per sec: 144348.68323451982, Loss: 10.076998710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 10.264368809544449, Tokens per sec: 144995.04810465127, Loss: 10.077482223510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 10.277492418339639, Tokens per sec: 145180.4329367703, Loss: 10.073884010314941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 10.292382641538586, Tokens per sec: 145390.7730628005, Loss: 10.076943397521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 10.365587753491566, Tokens per sec: 146424.8725701427, Loss: 10.073410987854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 10.382412169166747, Tokens per sec: 146662.5352073121, Loss: 10.072709083557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 10.362354881342437, Tokens per sec: 146379.2048372819, Loss: 10.06981086730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 9.911361585180442, Tokens per sec: 140008.44830220123, Loss: 10.068305015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 10.352751533219609, Tokens per sec: 146243.54740438255, Loss: 10.06562328338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 10.284242133056368, Tokens per sec: 145275.77978451163, Loss: 10.06148624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 10.157550102447637, Tokens per sec: 143486.1210715952, Loss: 10.060616493225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 10.169511480392456, Tokens per sec: 143655.088165694, Loss: 10.057272911071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 9.49249459276985, Tokens per sec: 134091.50973140958, Loss: 10.059510231018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 10.322963524619295, Tokens per sec: 145822.7603282276, Loss: 10.05810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 10.241256009576817, Tokens per sec: 144668.55539912573, Loss: 10.055248260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 10.18217285557744, Tokens per sec: 143833.94345997286, Loss: 10.056073188781738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 10.219994458197622, Tokens per sec: 144368.21353473957, Loss: 10.05279541015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 10.299046977902947, Tokens per sec: 145484.91385115933, Loss: 10.04677677154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 10.281601645935874, Tokens per sec: 145238.48011570983, Loss: 10.047637939453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 10.309066090846326, Tokens per sec: 145626.44440117624, Loss: 10.041624069213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 10.197820181127707, Tokens per sec: 144054.97845617842, Loss: 10.046454429626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 10.226524890230056, Tokens per sec: 144460.4627830135, Loss: 10.037193298339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 10.145078561922729, Tokens per sec: 143309.94739234602, Loss: 10.036327362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 10.108135946758281, Tokens per sec: 142788.0939435592, Loss: 10.035882949829102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 10.167304036138786, Tokens per sec: 143623.9056846581, Loss: 10.03288745880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 10.168727658637856, Tokens per sec: 143644.01585573296, Loss: 10.036385536193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 10.2099702136266, Tokens per sec: 144226.61049506342, Loss: 10.032857894897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 9.446288351339655, Tokens per sec: 133438.79777968428, Loss: 10.028310775756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 10.183148195198875, Tokens per sec: 143847.7211620364, Loss: 10.030346870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 10.20700328262538, Tokens per sec: 144184.69946173762, Loss: 10.03539752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 10.164259209749982, Tokens per sec: 143580.89429672898, Loss: 10.029635429382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 10.15346675168462, Tokens per sec: 143428.43943024956, Loss: 10.023759841918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 10.173003581871217, Tokens per sec: 143704.41778656933, Loss: 10.028903007507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 10.170322436476313, Tokens per sec: 143666.54377670833, Loss: 10.021859169006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 10.117211524697234, Tokens per sec: 142916.2960653119, Loss: 10.023637771606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 10.250203065630558, Tokens per sec: 144794.94201353705, Loss: 10.014778137207031 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 9.77957950420455, Tokens per sec: 138146.8872530052, Loss: 10.015514373779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 10.276650214102935, Tokens per sec: 145168.53591260052, Loss: 10.015929222106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 10.3216296209713, Tokens per sec: 145803.91753066285, Loss: 10.015279769897461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 10.250025891379522, Tokens per sec: 144792.43924015382, Loss: 10.008294105529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 10.165744210131958, Tokens per sec: 143601.87149521193, Loss: 10.01150894165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 9.132581026171543, Tokens per sec: 129007.3505521415, Loss: 10.014192581176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 10.15049094678058, Tokens per sec: 143386.40304367602, Loss: 10.010782241821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 10.284103296277227, Tokens per sec: 145273.81856840148, Loss: 10.002486228942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 10.195351212616941, Tokens per sec: 144020.10166885424, Loss: 10.009313583374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 10.167535366129751, Tokens per sec: 143627.17346505358, Loss: 10.007050514221191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 10.156003066277117, Tokens per sec: 143464.26755209232, Loss: 9.997225761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 10.146789013957513, Tokens per sec: 143334.10933349075, Loss: 9.994782447814941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 10.087941806489175, Tokens per sec: 142502.83038823676, Loss: 9.997943878173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 10.057205093888511, Tokens per sec: 142068.641866292, Loss: 9.997847557067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 10.071795549468618, Tokens per sec: 142274.7474581662, Loss: 9.996935844421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 10.10827216557291, Tokens per sec: 142790.0181781562, Loss: 9.988859176635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 10.136347947270403, Tokens per sec: 143186.61824128122, Loss: 9.989666938781738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 10.153088426921576, Tokens per sec: 143423.09519347915, Loss: 9.992227554321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 10.080884347210151, Tokens per sec: 142403.13632358986, Loss: 9.993131637573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 10.18846308252805, Tokens per sec: 143922.79955782124, Loss: 9.996275901794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 9.417094405083873, Tokens per sec: 133026.40246144647, Loss: 9.987053871154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 10.187530387169057, Tokens per sec: 143909.62425099494, Loss: 9.982683181762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 10.193038438100118, Tokens per sec: 143987.4312866277, Loss: 9.989241600036621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 10.0655390365087, Tokens per sec: 142186.36760603666, Loss: 9.986342430114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 10.049455744066323, Tokens per sec: 141959.17411712042, Loss: 9.978650093078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 10.19720667444879, Tokens per sec: 144046.31202650577, Loss: 9.976110458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 10.188080984215071, Tokens per sec: 143917.40201567337, Loss: 9.976738929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 10.299720688874453, Tokens per sec: 145494.43073003757, Loss: 9.979630470275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 10.15557172894455, Tokens per sec: 143458.17445679876, Loss: 9.970749855041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 10.18759418042658, Tokens per sec: 143910.52539812, Loss: 9.971717834472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 10.140997433805268, Tokens per sec: 143252.29714821582, Loss: 9.970973014831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 10.198492187228728, Tokens per sec: 144064.47125196081, Loss: 9.96383285522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 10.189901277093787, Tokens per sec: 143943.1155747252, Loss: 9.96531867980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 10.102178672249499, Tokens per sec: 142703.94115053304, Loss: 9.962735176086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 10.174712916424552, Tokens per sec: 143728.5639421086, Loss: 9.963194847106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 9.436835621115446, Tokens per sec: 133305.267983648, Loss: 9.958473205566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 10.107684093745416, Tokens per sec: 142781.71104261783, Loss: 9.961335182189941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 10.149534572931692, Tokens per sec: 143372.89325317662, Loss: 9.95206356048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 10.217837151150539, Tokens per sec: 144337.73929468912, Loss: 9.95479679107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 10.158855793901838, Tokens per sec: 143504.56534213191, Loss: 9.952436447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 10.07709133656102, Tokens per sec: 142349.55604293774, Loss: 9.943949699401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 10.120310910965586, Tokens per sec: 142960.0781691522, Loss: 9.944037437438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 10.178365631658945, Tokens per sec: 143780.1624018825, Loss: 9.941800117492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 10.170341163538733, Tokens per sec: 143666.80831623985, Loss: 9.937114715576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 10.170855614960107, Tokens per sec: 143674.07548580223, Loss: 9.938299179077148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 10.170575079556885, Tokens per sec: 143670.11262700008, Loss: 9.928755760192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 10.283722636083642, Tokens per sec: 145268.44134120859, Loss: 9.931439399719238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 10.288060980873691, Tokens per sec: 145329.72504244724, Loss: 9.927105903625488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 10.306550719574085, Tokens per sec: 145590.9121258472, Loss: 9.927423477172852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 10.311798562281938, Tokens per sec: 145665.04344556, Loss: 9.921058654785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 9.547868190036542, Tokens per sec: 134873.72026460338, Loss: 9.919536590576172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 10.389667876084351, Tokens per sec: 146765.02972919377, Loss: 9.912516593933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 9.884343151245611, Tokens per sec: 139626.78439273118, Loss: 9.915449142456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 10.282279726819699, Tokens per sec: 145248.058724214, Loss: 9.909289360046387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 10.275293463831556, Tokens per sec: 145149.37038235556, Loss: 9.907087326049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 10.156897542361092, Tokens per sec: 143476.90297130134, Loss: 9.899408340454102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 10.222907156588889, Tokens per sec: 144409.3584751828, Loss: 9.89836311340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 10.18935234156212, Tokens per sec: 143935.3612806914, Loss: 9.896757125854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 10.154697628820415, Tokens per sec: 143445.82686953852, Loss: 9.88761043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 10.184958665204107, Tokens per sec: 143873.2959626278, Loss: 9.890704154968262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 10.183145107789503, Tokens per sec: 143847.67754911885, Loss: 9.882471084594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 10.184708797049277, Tokens per sec: 143869.76631109233, Loss: 9.878613471984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 10.20603816658763, Tokens per sec: 144171.06617858927, Loss: 9.880475044250488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 10.230650600799812, Tokens per sec: 144518.74280135924, Loss: 9.872627258300781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 10.222187878734568, Tokens per sec: 144399.19791596837, Loss: 9.87063980102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 9.151705184653757, Tokens per sec: 129277.4994848758, Loss: 9.868154525756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 10.237465032601897, Tokens per sec: 144615.0038462693, Loss: 9.869701385498047 +------------------------------------------------------------------ + eval ppl=19237.65625, eval loss=9.864624977111816 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/model.pt +[rank0]:[2024-08-02 17:42:15,727] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010616070003379718, 'preprocessing_with_comm': 0.0005277139998725033, 'state_converting': 0.02839470600019922, : 0.03031319599995186}) +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000400, took 0.16s +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 10.205729936955981, Tokens per sec: 144166.71210956684, Loss: 9.855315208435059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 10.2817252781819, Tokens per sec: 145240.22655173455, Loss: 9.857254981994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 10.27425738322438, Tokens per sec: 145134.73464972893, Loss: 9.849400520324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 10.328665424069674, Tokens per sec: 145903.30567889026, Loss: 9.851024627685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 10.280362112982376, Tokens per sec: 145220.9704038557, Loss: 9.844959259033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 10.296027830760421, Tokens per sec: 145442.2652096999, Loss: 9.83842658996582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 10.283888937115067, Tokens per sec: 145270.7905188832, Loss: 9.836504936218262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 10.27591003322308, Tokens per sec: 145158.08007607787, Loss: 9.829629898071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 10.225649133329618, Tokens per sec: 144448.0917920385, Loss: 9.837343215942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 10.30703857360012, Tokens per sec: 145597.80357911484, Loss: 9.826777458190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 10.271534682190785, Tokens per sec: 145096.27362259067, Loss: 9.82370376586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 10.213613956332841, Tokens per sec: 144278.08220840263, Loss: 9.822669982910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 10.251871056776567, Tokens per sec: 144818.5041497912, Loss: 9.81297779083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 9.491098766087216, Tokens per sec: 134071.79220559288, Loss: 9.81275749206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 10.32735778136065, Tokens per sec: 145884.83384481876, Loss: 9.808419227600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 10.276754897048026, Tokens per sec: 145170.01466973993, Loss: 9.806970596313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 10.278899780994376, Tokens per sec: 145200.31342037435, Loss: 9.800897598266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 10.248129542313325, Tokens per sec: 144765.65130714222, Loss: 9.803784370422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 10.308334400397499, Tokens per sec: 145616.10850095734, Loss: 9.792960166931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 10.275801714810163, Tokens per sec: 145156.54996411648, Loss: 9.791539192199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 10.295011687797642, Tokens per sec: 145427.91111735353, Loss: 9.786686897277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 10.334905412863929, Tokens per sec: 145991.45210973112, Loss: 9.790814399719238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 10.318972871635777, Tokens per sec: 145766.38814090163, Loss: 9.787519454956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 10.27980081469653, Tokens per sec: 145213.04147286466, Loss: 9.776294708251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 10.26696015970502, Tokens per sec: 145031.65366200794, Loss: 9.777640342712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 10.166904853203807, Tokens per sec: 143618.26680418392, Loss: 9.775386810302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 10.109035192462878, Tokens per sec: 142800.79673869596, Loss: 9.773828506469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 10.092135864324339, Tokens per sec: 142562.07588387778, Loss: 9.762445449829102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 9.486921058944622, Tokens per sec: 134012.77768075, Loss: 9.761396408081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 10.101152438461291, Tokens per sec: 142689.44451462207, Loss: 9.759639739990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 10.09223738282417, Tokens per sec: 142563.50993988643, Loss: 9.759066581726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 10.178551231442954, Tokens per sec: 143782.78419481585, Loss: 9.753396987915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 10.425379200664358, Tokens per sec: 147269.48989829293, Loss: 9.748218536376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 10.418698590719414, Tokens per sec: 147175.11922842427, Loss: 9.7437105178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 10.373430210348943, Tokens per sec: 146535.65555450282, Loss: 9.740473747253418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 10.302652307741587, Tokens per sec: 145535.84294218154, Loss: 9.739056587219238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 10.354877993691366, Tokens per sec: 146273.58590398403, Loss: 9.737876892089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 10.212716382992442, Tokens per sec: 144265.40303717606, Loss: 9.729854583740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 10.026254823061894, Tokens per sec: 141631.43660889863, Loss: 9.734286308288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 10.104898601717252, Tokens per sec: 142742.36302637713, Loss: 9.729402542114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 10.27405922992896, Tokens per sec: 145131.93552519096, Loss: 9.723472595214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 10.230861521324432, Tokens per sec: 144521.72227649123, Loss: 9.727094650268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 10.292955962889534, Tokens per sec: 145398.8718322819, Loss: 9.718483924865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 9.492732299051534, Tokens per sec: 134094.86758364452, Loss: 9.719978332519531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 10.224118732115405, Tokens per sec: 144426.47325885916, Loss: 9.706987380981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 10.319618520412648, Tokens per sec: 145775.5086116484, Loss: 9.711590766906738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 10.280723552301218, Tokens per sec: 145226.07611589652, Loss: 9.700068473815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 10.28840459887484, Tokens per sec: 145334.57900955723, Loss: 9.706768989562988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 10.305255185795385, Tokens per sec: 145572.6113432029, Loss: 9.694439888000488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 10.300463454966172, Tokens per sec: 145504.92308540764, Loss: 9.694628715515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 10.03985822737174, Tokens per sec: 141823.59905930268, Loss: 9.693022727966309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 10.257633603409626, Tokens per sec: 144899.9062059497, Loss: 9.693314552307129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 10.260078780929328, Tokens per sec: 144934.446920403, Loss: 9.684147834777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 10.23177062001722, Tokens per sec: 144534.56425549046, Loss: 9.684213638305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 10.262433128892638, Tokens per sec: 144967.70457146066, Loss: 9.682662010192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 10.282077552240665, Tokens per sec: 145245.20279479874, Loss: 9.679064750671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 10.253124462176801, Tokens per sec: 144836.20982460413, Loss: 9.67774486541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 10.300670553173068, Tokens per sec: 145507.84856626563, Loss: 9.674714088439941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 10.343486265880099, Tokens per sec: 146112.66571954425, Loss: 9.667888641357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 9.593188727991413, Tokens per sec: 135513.92071948032, Loss: 9.667743682861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 10.30350623386425, Tokens per sec: 145547.90555036822, Loss: 9.66889762878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 10.283443174384828, Tokens per sec: 145264.493649615, Loss: 9.657283782958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 10.338706771064208, Tokens per sec: 146045.1503083578, Loss: 9.663376808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 10.355983444648494, Tokens per sec: 146289.20156605545, Loss: 9.65365982055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 10.33070576671392, Tokens per sec: 145932.12767324195, Loss: 9.65356731414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 10.359842828805936, Tokens per sec: 146343.71944259963, Loss: 9.654111862182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 10.356181531317054, Tokens per sec: 146291.99974941867, Loss: 9.649432182312012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 10.361416989290875, Tokens per sec: 146365.95612168562, Loss: 9.642577171325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 10.312003993651132, Tokens per sec: 145667.94538057534, Loss: 9.637773513793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 10.148422434347863, Tokens per sec: 143357.18312131622, Loss: 9.63598918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 10.31789949215048, Tokens per sec: 145751.22552223573, Loss: 9.643247604370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 10.249710027043022, Tokens per sec: 144787.97732286353, Loss: 9.630722045898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 10.167890098489792, Tokens per sec: 143632.18443421915, Loss: 9.631145477294922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 10.182419824610962, Tokens per sec: 143837.43215835857, Loss: 9.630815505981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 9.084089104148832, Tokens per sec: 128322.35094848035, Loss: 9.626769065856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 10.154243262687409, Tokens per sec: 143439.40846812146, Loss: 9.6152925491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 10.26834475092462, Tokens per sec: 145051.21247504518, Loss: 9.616559982299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 10.273198362355172, Tokens per sec: 145119.7748616773, Loss: 9.61669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 10.127812667100265, Tokens per sec: 143066.04839604095, Loss: 9.613576889038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 10.175704629687884, Tokens per sec: 143742.5729391537, Loss: 9.607438087463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 10.17104232075391, Tokens per sec: 143676.71290229188, Loss: 9.611551284790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 10.216042419917207, Tokens per sec: 144312.38682087016, Loss: 9.603131294250488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 10.134866112887245, Tokens per sec: 143165.6857658747, Loss: 9.609889030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 10.194221785586423, Tokens per sec: 144004.14731942865, Loss: 9.60080623626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 10.15796406280061, Tokens per sec: 143491.96869870278, Loss: 9.601019859313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 10.25435224143131, Tokens per sec: 144853.55350304872, Loss: 9.594995498657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 10.25111788379533, Tokens per sec: 144807.8647861175, Loss: 9.595114707946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 10.265563355209789, Tokens per sec: 145011.92232355583, Loss: 9.590287208557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 9.458000031451435, Tokens per sec: 133604.2376282227, Loss: 9.588107109069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 10.203701521078804, Tokens per sec: 144138.05859339406, Loss: 9.581729888916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 10.256525621242078, Tokens per sec: 144884.25478785826, Loss: 9.585355758666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 10.269311432807337, Tokens per sec: 145064.8678774072, Loss: 9.585918426513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 10.208583616733081, Tokens per sec: 144207.02334976423, Loss: 9.573511123657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 10.25200525318717, Tokens per sec: 144820.3998157958, Loss: 9.570962905883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 10.275992411625467, Tokens per sec: 145159.24375799912, Loss: 9.569580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 10.220627420459603, Tokens per sec: 144377.15479506712, Loss: 9.57296371459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 10.185977830796116, Tokens per sec: 143887.69275280382, Loss: 9.567734718322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 10.139864268013795, Tokens per sec: 143236.2899848438, Loss: 9.563718795776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 10.231872724008454, Tokens per sec: 144536.00658218344, Loss: 9.56546688079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 10.040559768890724, Tokens per sec: 141833.50907405175, Loss: 9.56107234954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 10.316426480724605, Tokens per sec: 145730.4176804175, Loss: 9.556859970092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 10.255912606719258, Tokens per sec: 144875.59531041016, Loss: 9.551627159118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 10.242426557934413, Tokens per sec: 144685.09063071842, Loss: 9.555253028869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 8.949773842546364, Tokens per sec: 126425.00604801948, Loss: 9.55272102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 10.246764681306027, Tokens per sec: 144746.37120418792, Loss: 9.547101020812988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 10.284943755223589, Tokens per sec: 145285.69093850223, Loss: 9.55477237701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 10.244332085137343, Tokens per sec: 144712.008214603, Loss: 9.547393798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 10.249201705250252, Tokens per sec: 144780.79674077776, Loss: 9.541619300842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 10.318139209896259, Tokens per sec: 145754.61178852507, Loss: 9.539627075195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 10.263946902021182, Tokens per sec: 144989.08821537148, Loss: 9.540337562561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 10.383058471053621, Tokens per sec: 146671.66490392727, Loss: 9.535943984985352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 10.267867384245749, Tokens per sec: 145044.46916663036, Loss: 9.542598724365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 10.322103317394378, Tokens per sec: 145810.608992839, Loss: 9.535621643066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 10.333374322921173, Tokens per sec: 145969.82384753384, Loss: 9.528502464294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 10.341420193801275, Tokens per sec: 146083.48026976013, Loss: 9.53730583190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 10.394957087773253, Tokens per sec: 146839.74542944756, Loss: 9.53094482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 10.292577272471089, Tokens per sec: 145393.52243024285, Loss: 9.523018836975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 9.544841274818232, Tokens per sec: 134830.96189087082, Loss: 9.525299072265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 10.365555301065775, Tokens per sec: 146424.41414536026, Loss: 9.52488899230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 10.347761896221147, Tokens per sec: 146173.06351296743, Loss: 9.519216537475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 10.297198607144365, Tokens per sec: 145458.8036624055, Loss: 9.516359329223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 10.221647803463041, Tokens per sec: 144391.56878246507, Loss: 9.52022933959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 10.327557096481096, Tokens per sec: 145887.64938136202, Loss: 9.519647598266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 10.317567586608531, Tokens per sec: 145746.53700597974, Loss: 9.51441478729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 10.280490308167185, Tokens per sec: 145222.78129620897, Loss: 9.507948875427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 9.664601245694985, Tokens per sec: 136522.6979401581, Loss: 9.508354187011719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 10.365732025949816, Tokens per sec: 146426.91057096014, Loss: 9.506451606750488 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 10.272904942509864, Tokens per sec: 145115.62999652533, Loss: 9.51048469543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 10.241409044281388, Tokens per sec: 144670.71717592495, Loss: 9.506376266479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 10.250879154180428, Tokens per sec: 144804.4924782203, Loss: 9.504505157470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 10.261652406446945, Tokens per sec: 144956.67604251063, Loss: 9.505819320678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 10.27619732749339, Tokens per sec: 145162.13841101463, Loss: 9.49921989440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 9.579644694250192, Tokens per sec: 135322.5969410505, Loss: 9.498015403747559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 10.300166485398163, Tokens per sec: 145500.72807667425, Loss: 9.49728012084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 10.163870995342831, Tokens per sec: 143575.4103582926, Loss: 9.502933502197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 10.217108696509849, Tokens per sec: 144327.449103677, Loss: 9.489360809326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 10.268866379387335, Tokens per sec: 145058.5810278951, Loss: 9.486632347106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 10.332387412972222, Tokens per sec: 145955.8827023764, Loss: 9.490132331848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 10.260360932933564, Tokens per sec: 144938.43261540143, Loss: 9.490154266357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 10.24652344403931, Tokens per sec: 144742.9634730608, Loss: 9.488014221191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 10.217286522246418, Tokens per sec: 144329.9610799813, Loss: 9.485597610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 10.25241434872131, Tokens per sec: 144826.1787222003, Loss: 9.485361099243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 10.230300040553395, Tokens per sec: 144513.79076770417, Loss: 9.482256889343262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 10.252758809845705, Tokens per sec: 144831.04459931646, Loss: 9.478363037109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 10.315807411499923, Tokens per sec: 145721.67267391214, Loss: 9.48255729675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 10.198815670232786, Tokens per sec: 144069.04079097533, Loss: 9.473245620727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 10.234884473805435, Tokens per sec: 144578.5507283258, Loss: 9.476582527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 9.439024328575533, Tokens per sec: 133336.1857876894, Loss: 9.475544929504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 10.224807811182913, Tokens per sec: 144436.20722832138, Loss: 9.469179153442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 10.217569847106244, Tokens per sec: 144333.9633428035, Loss: 9.470986366271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 10.244621274013706, Tokens per sec: 144716.09331285092, Loss: 9.468491554260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 10.281656498223795, Tokens per sec: 145239.25496219806, Loss: 9.4631929397583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 10.245666028329357, Tokens per sec: 144730.8515707688, Loss: 9.464640617370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 10.19838424180724, Tokens per sec: 144062.94640889723, Loss: 9.463068008422852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 10.241436960336545, Tokens per sec: 144671.11151968202, Loss: 9.457244873046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 10.186345762145422, Tokens per sec: 143892.8901716302, Loss: 9.46117877960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 10.230955951108555, Tokens per sec: 144523.05619690518, Loss: 9.460040092468262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 10.206967091065424, Tokens per sec: 144184.18821773675, Loss: 9.459478378295898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 10.139067317437942, Tokens per sec: 143225.03221641915, Loss: 9.452197074890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 10.209679237333447, Tokens per sec: 144222.500147666, Loss: 9.454987525939941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 10.23803796131078, Tokens per sec: 144623.09706926692, Loss: 9.451032638549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 10.223626895450703, Tokens per sec: 144419.52554661484, Loss: 9.44702434539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 9.44719069112948, Tokens per sec: 133451.54428204207, Loss: 9.449645042419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 10.17831873132212, Tokens per sec: 143779.49988510038, Loss: 9.445639610290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 10.254277927755874, Tokens per sec: 144852.5037439116, Loss: 9.440186500549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 10.222413844882285, Tokens per sec: 144402.38992641406, Loss: 9.436376571655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 10.259339219025078, Tokens per sec: 144923.99982756432, Loss: 9.437812805175781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 10.235397317240343, Tokens per sec: 144585.79518338141, Loss: 9.440080642700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 10.253235369750213, Tokens per sec: 144837.7765111915, Loss: 9.43734073638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 10.253556282483904, Tokens per sec: 144842.309742423, Loss: 9.439736366271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 10.25834578027626, Tokens per sec: 144909.9664562147, Loss: 9.437381744384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 10.277566259913655, Tokens per sec: 145181.47602697634, Loss: 9.427556037902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 10.282266789883149, Tokens per sec: 145247.87597632446, Loss: 9.432079315185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 10.142939616751066, Tokens per sec: 143279.73253317512, Loss: 9.429582595825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 10.132679954762088, Tokens per sec: 143134.80397388115, Loss: 9.429387092590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 10.17658568316969, Tokens per sec: 143755.0187499335, Loss: 9.427623748779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 9.444077040895616, Tokens per sec: 133407.56068462686, Loss: 9.423463821411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 10.207813594299907, Tokens per sec: 144196.14596978974, Loss: 9.427388191223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 10.242683834261811, Tokens per sec: 144688.7249305141, Loss: 9.42039680480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 10.221118159484863, Tokens per sec: 144384.08700199527, Loss: 9.419669151306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 10.198900460148867, Tokens per sec: 144070.23853807396, Loss: 9.417304992675781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 10.261962863373151, Tokens per sec: 144961.06157442057, Loss: 9.415884017944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 10.185837215167991, Tokens per sec: 143885.7064085733, Loss: 9.419095039367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 9.829127039558472, Tokens per sec: 138846.7985096456, Loss: 9.419087409973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 10.102325760800394, Tokens per sec: 142706.01893163275, Loss: 9.414474487304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 10.147584485207565, Tokens per sec: 143345.34620487635, Loss: 9.41500186920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 10.129995753910363, Tokens per sec: 143096.88680246612, Loss: 9.412477493286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 10.168045503528244, Tokens per sec: 143634.37969448735, Loss: 9.409485816955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 10.19525686649412, Tokens per sec: 144018.76893024452, Loss: 9.406645774841309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 10.159603792349982, Tokens per sec: 143515.1316100616, Loss: 9.404681205749512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 10.230557509596265, Tokens per sec: 144517.4277898094, Loss: 9.409933090209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 10.210049313540141, Tokens per sec: 144227.72786486783, Loss: 9.40280532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 9.427043396561123, Tokens per sec: 133166.9424717095, Loss: 9.397782325744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 10.17053046734725, Tokens per sec: 143669.4824324307, Loss: 9.402655601501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 9.927715711433525, Tokens per sec: 140239.46760468092, Loss: 9.400113105773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 9.879957603952986, Tokens per sec: 139564.83390630007, Loss: 9.399343490600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 10.208593678578758, Tokens per sec: 144207.16548395922, Loss: 9.393732070922852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 10.199854531162064, Tokens per sec: 144083.71579858754, Loss: 9.396077156066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 9.962495489221084, Tokens per sec: 140730.76869166904, Loss: 9.395782470703125 +------------------------------------------------------------------ +[rank0]:[2024-08-02 17:46:04,722] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010460220000823028, 'preprocessing_with_comm': 0.0005041619997427915, 'state_converting': 0.033199113000137004, : 0.035121100999731425}) +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 10.196014162337137, Tokens per sec: 144029.4665336944, Loss: 9.391682624816895 +------------------------------------------------------------------ + eval ppl=11809.18359375, eval loss=9.376632690429688 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000600, took 0.16s +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 10.099458117681554, Tokens per sec: 142665.5104444887, Loss: 9.38862133026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 10.166650275599396, Tokens per sec: 143614.67062670013, Loss: 9.386565208435059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 10.15172148380729, Tokens per sec: 143403.785678565, Loss: 9.387913703918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 10.126603343977266, Tokens per sec: 143048.9654299427, Loss: 9.384979248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 10.207236539214687, Tokens per sec: 144187.99445736915, Loss: 9.382861137390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 10.200256171449892, Tokens per sec: 144089.38939176555, Loss: 9.381598472595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 9.396122198885157, Tokens per sec: 132730.1479032689, Loss: 9.38084888458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 10.200071104295853, Tokens per sec: 144086.7751227929, Loss: 9.37721061706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 10.225700294959642, Tokens per sec: 144448.81450408677, Loss: 9.37877082824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 10.168374353679525, Tokens per sec: 143639.0250501238, Loss: 9.373688697814941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 10.156970460161851, Tokens per sec: 143477.9330122347, Loss: 9.372486114501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 10.147609659146072, Tokens per sec: 143345.70181334025, Loss: 9.372447967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 10.161427827755542, Tokens per sec: 143540.89803625704, Loss: 9.375641822814941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 10.131526317290264, Tokens per sec: 143118.50762640705, Loss: 9.371562957763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 10.056136478294686, Tokens per sec: 142053.54654262358, Loss: 9.373139381408691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 10.152547823089417, Tokens per sec: 143415.4585934988, Loss: 9.364054679870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 10.204916622424806, Tokens per sec: 144155.22318299377, Loss: 9.361952781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 10.192612486665885, Tokens per sec: 143981.41427283536, Loss: 9.366772651672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 10.25253958654045, Tokens per sec: 144827.94783864068, Loss: 9.358394622802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 10.278904135937447, Tokens per sec: 145200.37493854383, Loss: 9.359618186950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 10.251904546763159, Tokens per sec: 144818.9772312156, Loss: 9.357376098632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 9.503832636920148, Tokens per sec: 134251.67157745085, Loss: 9.360281944274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 10.312386915194207, Tokens per sec: 145673.35455171764, Loss: 9.357015609741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 10.31771545379113, Tokens per sec: 145748.62578607447, Loss: 9.360085487365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 10.290595382101055, Tokens per sec: 145365.5261359871, Loss: 9.354413986206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 10.23794818104477, Tokens per sec: 144621.82882820594, Loss: 9.348170280456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 10.251683994397474, Tokens per sec: 144815.86169615784, Loss: 9.35173511505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 10.294203796400762, Tokens per sec: 145416.49879827892, Loss: 9.352083206176758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 10.233720442976844, Tokens per sec: 144562.10756371255, Loss: 9.350330352783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 10.247738545725959, Tokens per sec: 144760.12806747245, Loss: 9.346280097961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 10.261768663621613, Tokens per sec: 144958.3182978647, Loss: 9.344816207885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 10.27532500828846, Tokens per sec: 145149.8159811187, Loss: 9.34606647491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 10.230421460430913, Tokens per sec: 144515.50595168755, Loss: 9.336515426635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 10.196415017126194, Tokens per sec: 144035.1290308728, Loss: 9.339628219604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 10.189500862810947, Tokens per sec: 143937.45930016224, Loss: 9.341111183166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 9.496247881574174, Tokens per sec: 134144.52889905946, Loss: 9.33900260925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 10.270922595571166, Tokens per sec: 145087.62725275513, Loss: 9.33552360534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 10.267088664891276, Tokens per sec: 145033.468933459, Loss: 9.334668159484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 10.292544974765281, Tokens per sec: 145393.06619104353, Loss: 9.333837509155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 10.20882924094699, Tokens per sec: 144210.4930511528, Loss: 9.331905364990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 10.295961997263078, Tokens per sec: 145441.335242033, Loss: 9.325151443481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 10.213558858188348, Tokens per sec: 144277.3038889307, Loss: 9.325544357299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 10.127421027232067, Tokens per sec: 143060.51606935018, Loss: 9.329853057861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 10.270486089038863, Tokens per sec: 145081.46113705685, Loss: 9.33024787902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 10.249060848145948, Tokens per sec: 144778.80698544128, Loss: 9.320714950561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 10.312378267438232, Tokens per sec: 145673.23239303287, Loss: 9.325393676757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 10.266848510566343, Tokens per sec: 145030.07650002814, Loss: 9.32136058807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 10.216354317480615, Tokens per sec: 144316.79270330185, Loss: 9.318805694580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 10.359627184618397, Tokens per sec: 146340.67324074093, Loss: 9.320027351379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 10.315530644260823, Tokens per sec: 145717.7630444072, Loss: 9.319043159484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 9.517475577282466, Tokens per sec: 134444.39251634423, Loss: 9.319427490234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 10.304251583540006, Tokens per sec: 145558.434401591, Loss: 9.315238952636719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 10.272510836384944, Tokens per sec: 145110.0628313541, Loss: 9.30887222290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 10.337131190681612, Tokens per sec: 146022.89357171804, Loss: 9.309306144714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 10.343671611434932, Tokens per sec: 146115.2839212221, Loss: 9.304703712463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 10.386122301014456, Tokens per sec: 146714.94473737822, Loss: 9.304349899291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 10.371066323214412, Tokens per sec: 146502.26315257876, Loss: 9.302862167358398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 10.254712234163708, Tokens per sec: 144858.63878052853, Loss: 9.309221267700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 10.266417329491494, Tokens per sec: 145023.98561203983, Loss: 9.301545143127441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 10.292522048348266, Tokens per sec: 145392.742331194, Loss: 9.298178672790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 10.351714057314712, Tokens per sec: 146228.89196170226, Loss: 9.299586296081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 10.378630348999565, Tokens per sec: 146609.1130040341, Loss: 9.296051025390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 10.411033556985432, Tokens per sec: 147066.84253303226, Loss: 9.299639701843262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 10.393703264902891, Tokens per sec: 146822.03385742824, Loss: 9.298367500305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 9.598497919003401, Tokens per sec: 135588.91864876988, Loss: 9.294963836669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 10.401778957076598, Tokens per sec: 146936.11153691655, Loss: 9.289440155029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 10.405187944802947, Tokens per sec: 146984.267088275, Loss: 9.289616584777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 10.406297054240966, Tokens per sec: 146999.93443025218, Loss: 9.294499397277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 10.397615128337534, Tokens per sec: 146877.29305917842, Loss: 9.282642364501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 10.387806077053218, Tokens per sec: 146738.7298519106, Loss: 9.28408145904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 10.382206993129415, Tokens per sec: 146659.636879129, Loss: 9.28907299041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 10.34239376251176, Tokens per sec: 146097.23295583652, Loss: 9.28598403930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 10.364577413335377, Tokens per sec: 146410.60044856695, Loss: 9.28571891784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 10.376793360428683, Tokens per sec: 146583.16360070062, Loss: 9.280057907104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 10.393653912887878, Tokens per sec: 146821.33670810287, Loss: 9.277195930480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 10.395704968073366, Tokens per sec: 146850.31002841154, Loss: 9.275837898254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 10.407501339504346, Tokens per sec: 147016.94623126357, Loss: 9.27735710144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 10.38478091634768, Tokens per sec: 146695.99626252358, Loss: 9.276752471923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 10.379178343661067, Tokens per sec: 146616.85400728334, Loss: 9.273816108703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 9.509591860399652, Tokens per sec: 134333.02669055626, Loss: 9.271092414855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 10.350344191367737, Tokens per sec: 146209.54115868997, Loss: 9.266501426696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 10.366952557434612, Tokens per sec: 146444.15186700426, Loss: 9.264703750610352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 10.35696301577883, Tokens per sec: 146303.0390426505, Loss: 9.267648696899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 10.358402265101995, Tokens per sec: 146323.36995911688, Loss: 9.264509201049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 10.373722576065989, Tokens per sec: 146539.78552897996, Loss: 9.260058403015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 10.333963164822713, Tokens per sec: 145978.1418611812, Loss: 9.26374626159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 10.178446275222809, Tokens per sec: 143781.3015773777, Loss: 9.259567260742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 10.298535658283848, Tokens per sec: 145477.69092161112, Loss: 9.261046409606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 10.305797396738843, Tokens per sec: 145580.27064532743, Loss: 9.25600528717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 10.308764217310696, Tokens per sec: 145622.1801187218, Loss: 9.256509780883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 10.31310256039188, Tokens per sec: 145683.46379582095, Loss: 9.25269889831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 10.317642435708786, Tokens per sec: 145747.59432855784, Loss: 9.251874923706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 10.31298797433658, Tokens per sec: 145681.84514679262, Loss: 9.250983238220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 10.447674294320738, Tokens per sec: 147584.43163871428, Loss: 9.248920440673828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 9.659492477885665, Tokens per sec: 136450.5312002433, Loss: 9.244975090026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 10.48316876812523, Tokens per sec: 148085.8285616255, Loss: 9.247379302978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 10.452370290749228, Tokens per sec: 147650.76754700852, Loss: 9.248109817504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 10.45871229182464, Tokens per sec: 147740.35500904068, Loss: 9.246539115905762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 10.473688100233332, Tokens per sec: 147951.90411653236, Loss: 9.241764068603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 10.446702892370244, Tokens per sec: 147570.70956041076, Loss: 9.237360954284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 10.352347412912579, Tokens per sec: 146237.83877831453, Loss: 9.242709159851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 10.427135148995362, Tokens per sec: 147294.29452266367, Loss: 9.23869800567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 10.486925210364442, Tokens per sec: 148138.89227487307, Loss: 9.23484992980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 10.376727249824212, Tokens per sec: 146582.22971860293, Loss: 9.24294662475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 9.95510263385284, Tokens per sec: 140626.33680310007, Loss: 9.233550071716309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 10.445406183749219, Tokens per sec: 147552.3921818781, Loss: 9.23662281036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 10.429525968872204, Tokens per sec: 147328.06737801252, Loss: 9.224603652954102 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 10.42055979147194, Tokens per sec: 147201.41065439038, Loss: 9.230449676513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 10.465548532384322, Tokens per sec: 147836.9241256808, Loss: 9.227519989013672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 9.584312845044973, Tokens per sec: 135388.53950036498, Loss: 9.22962760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 10.317995700380289, Tokens per sec: 145752.584565073, Loss: 9.223089218139648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 9.704381150724155, Tokens per sec: 137084.631104323, Loss: 9.2258939743042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 10.275843793439215, Tokens per sec: 145157.14436918477, Loss: 9.219230651855469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 10.287051991843494, Tokens per sec: 145315.47200695286, Loss: 9.222535133361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 10.281498055845697, Tokens per sec: 145237.0167963, Loss: 9.220541954040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 10.263780220981944, Tokens per sec: 144986.73366968523, Loss: 9.223250389099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 10.269219088648873, Tokens per sec: 145063.563418658, Loss: 9.21098518371582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 10.385740112116755, Tokens per sec: 146709.54591562544, Loss: 9.21323013305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 10.367374291136915, Tokens per sec: 146450.10930087938, Loss: 9.210025787353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 10.308208819864396, Tokens per sec: 145614.33454332035, Loss: 9.213190078735352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 10.276377179682964, Tokens per sec: 145164.67901310744, Loss: 9.207383155822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 10.258213328466068, Tokens per sec: 144908.0954345331, Loss: 9.20666217803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 10.325162611915493, Tokens per sec: 145853.82475841467, Loss: 9.206585884094238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 10.295310758965446, Tokens per sec: 145432.13581340326, Loss: 9.201839447021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 10.313433025447155, Tokens per sec: 145688.1319636816, Loss: 9.198698043823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 9.511503754040037, Tokens per sec: 134360.03420709257, Loss: 9.202643394470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 10.290527294418453, Tokens per sec: 145364.56432557313, Loss: 9.203864097595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 10.333923172691543, Tokens per sec: 145977.57693009876, Loss: 9.199012756347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 10.34085126990215, Tokens per sec: 146075.44361892925, Loss: 9.195448875427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 10.267127125460329, Tokens per sec: 145034.01222960933, Loss: 9.199257850646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 10.2301624672581, Tokens per sec: 144511.84739963748, Loss: 9.198324203491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 10.256221032820386, Tokens per sec: 144879.95215477142, Loss: 9.19243335723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 10.349194957943304, Tokens per sec: 146193.30702303577, Loss: 9.192919731140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 10.357725619283336, Tokens per sec: 146313.8116224235, Loss: 9.192170143127441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 10.277046273838943, Tokens per sec: 145174.13067459123, Loss: 9.186891555786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 10.44709417385213, Tokens per sec: 147576.23682451644, Loss: 9.18764591217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 10.497401930708783, Tokens per sec: 148286.88701263955, Loss: 9.188284873962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 10.435149215762412, Tokens per sec: 147407.5016782095, Loss: 9.190230369567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 10.31933078162967, Tokens per sec: 145771.44399750078, Loss: 9.187028884887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 10.437792200141816, Tokens per sec: 147444.83662343043, Loss: 9.181229591369629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 9.602081589453961, Tokens per sec: 135639.5417780642, Loss: 9.183816909790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 10.337545719716998, Tokens per sec: 146028.74923206517, Loss: 9.177844047546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 10.41024071173269, Tokens per sec: 147055.64275663157, Loss: 9.182524681091309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 10.442664218641013, Tokens per sec: 147513.65902934407, Loss: 9.175487518310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 10.360314996247498, Tokens per sec: 146350.38930629732, Loss: 9.175530433654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 10.394999301783876, Tokens per sec: 146840.34174692354, Loss: 9.174867630004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 10.432682079810274, Tokens per sec: 147372.65077771104, Loss: 9.17035961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 10.435562394293983, Tokens per sec: 147413.3382612642, Loss: 9.169294357299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 10.447340918641828, Tokens per sec: 147579.7223552213, Loss: 9.171226501464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 10.40845043671041, Tokens per sec: 147030.35323149932, Loss: 9.16797924041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 10.450972294134228, Tokens per sec: 147631.01936860578, Loss: 9.169792175292969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 10.435275893633118, Tokens per sec: 147409.29113689886, Loss: 9.170798301696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 10.390752970215788, Tokens per sec: 146780.35782960648, Loss: 9.163594245910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 10.3250239340448, Tokens per sec: 145851.86578705412, Loss: 9.16364860534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 10.271596081891895, Tokens per sec: 145097.14095820393, Loss: 9.168484687805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 9.612575471375706, Tokens per sec: 135787.77894123326, Loss: 9.169190406799316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 10.380784683853882, Tokens per sec: 146639.54525872326, Loss: 9.162364959716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 10.390045347120463, Tokens per sec: 146770.36190616974, Loss: 9.162246704101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 10.384751164731858, Tokens per sec: 146695.57598953389, Loss: 9.163891792297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 10.371849713298046, Tokens per sec: 146513.32936473156, Loss: 9.158731460571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 10.36796480372268, Tokens per sec: 146458.45091470564, Loss: 9.157739639282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 10.357043680483121, Tokens per sec: 146304.1785167766, Loss: 9.14681339263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 10.367459085949967, Tokens per sec: 146451.30711715293, Loss: 9.154752731323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 10.368489794194765, Tokens per sec: 146465.86695948802, Loss: 9.156282424926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 10.301760596883655, Tokens per sec: 145523.24658471034, Loss: 9.1522855758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 10.2976154601812, Tokens per sec: 145464.69215172718, Loss: 9.155441284179688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 10.291457700571563, Tokens per sec: 145377.70729494904, Loss: 9.156362533569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 10.208285994019688, Tokens per sec: 144202.81911466256, Loss: 9.149214744567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 10.283742975232036, Tokens per sec: 145268.7286531568, Loss: 9.147082328796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 10.24668328583703, Tokens per sec: 144745.22140723866, Loss: 9.153770446777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 9.564722172967306, Tokens per sec: 135111.80057047945, Loss: 9.145647048950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 10.402948103592628, Tokens per sec: 146952.62696601683, Loss: 9.143547058105469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 10.3308569036416, Tokens per sec: 145934.2626419386, Loss: 9.144509315490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 10.33262008730231, Tokens per sec: 145959.1694729816, Loss: 9.139680862426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 10.300451723028127, Tokens per sec: 145504.75735939443, Loss: 9.14000415802002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 10.332452224861647, Tokens per sec: 145956.79823875273, Loss: 9.1375732421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 10.380907328223522, Tokens per sec: 146641.27773995086, Loss: 9.140440940856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 10.374273108785152, Tokens per sec: 146547.56238497366, Loss: 9.141499519348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 10.086811159327118, Tokens per sec: 142486.85880316477, Loss: 9.135473251342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 10.395335519780028, Tokens per sec: 146845.09118114886, Loss: 9.136520385742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 10.392735757175942, Tokens per sec: 146808.3667891444, Loss: 9.135398864746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 10.376000533459422, Tokens per sec: 146571.96408257342, Loss: 9.133368492126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 10.40371936641867, Tokens per sec: 146963.5218678504, Loss: 9.13526725769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 10.408852311072554, Tokens per sec: 147036.0301312242, Loss: 9.130904197692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 9.601714638793794, Tokens per sec: 135634.35821250748, Loss: 9.132869720458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 10.377847343002736, Tokens per sec: 146598.05221751303, Loss: 9.130097389221191 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 10.42761784936887, Tokens per sec: 147301.1131751474, Loss: 9.131654739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 10.369626717264078, Tokens per sec: 146481.92719837703, Loss: 9.127538681030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 10.371468435219544, Tokens per sec: 146507.9434092618, Loss: 9.133221626281738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 10.397662917854042, Tokens per sec: 146877.9681365625, Loss: 9.123183250427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 10.350316406697997, Tokens per sec: 146209.14867089063, Loss: 9.128449440002441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 10.251047550495084, Tokens per sec: 144806.87125398303, Loss: 9.12617301940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 10.358471798696938, Tokens per sec: 146324.3521945695, Loss: 9.120492935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 10.286407598123391, Tokens per sec: 145306.36926520822, Loss: 9.120668411254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 10.323996134989143, Tokens per sec: 145837.3470400876, Loss: 9.127528190612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 10.377950956008897, Tokens per sec: 146599.5158606353, Loss: 9.117142677307129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 10.325131558829776, Tokens per sec: 145853.3861007884, Loss: 9.123003959655762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 10.266080701645459, Tokens per sec: 145019.23038825206, Loss: 9.119080543518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 9.510346197578167, Tokens per sec: 134343.68249974586, Loss: 9.118438720703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 10.101107853724706, Tokens per sec: 142688.81470813853, Loss: 9.116881370544434 +------------------------------------------------------------------ + eval ppl=9070.4287109375, eval loss=9.112774848937988 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0000800, took 0.17s +[rank0]:[2024-08-02 17:49:51,987] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010285759999533184, 'preprocessing_with_comm': 0.0005046030000812607, 'state_converting': 0.028110265000123036, : 0.02996069400023771}) +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 10.201942578565212, Tokens per sec: 144113.2116730326, Loss: 9.116273880004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 10.355764094476609, Tokens per sec: 146286.10301325456, Loss: 9.112004280090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 10.403796877142858, Tokens per sec: 146964.61678867706, Loss: 9.115842819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 10.205280169672887, Tokens per sec: 144160.35867176097, Loss: 9.109709739685059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 10.2295683581182, Tokens per sec: 144503.4549806863, Loss: 9.113845825195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 10.189900615904259, Tokens per sec: 143943.1062347249, Loss: 9.108858108520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 10.14856016655319, Tokens per sec: 143359.12873415495, Loss: 9.109667778015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 10.176175995798534, Tokens per sec: 143749.23148320615, Loss: 9.107152938842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 10.214764591387608, Tokens per sec: 144294.33614356534, Loss: 9.106344223022461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 10.198123896977147, Tokens per sec: 144059.2687632608, Loss: 9.105679512023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 10.244018106745184, Tokens per sec: 144707.572938268, Loss: 9.109625816345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 10.216510165626882, Tokens per sec: 144318.99422293584, Loss: 9.114452362060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 10.21563126477792, Tokens per sec: 144306.5788203681, Loss: 9.104471206665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 9.455259776248308, Tokens per sec: 133565.5286299035, Loss: 9.104260444641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 10.226479434868242, Tokens per sec: 144459.82067802924, Loss: 9.104790687561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 10.247461506202884, Tokens per sec: 144756.2145916689, Loss: 9.1082124710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 10.244620948894003, Tokens per sec: 144716.0887201918, Loss: 9.103623390197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 10.247224139583947, Tokens per sec: 144752.86153752892, Loss: 9.099392890930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 10.242660416243377, Tokens per sec: 144688.3941262754, Loss: 9.100199699401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 10.22643252203752, Tokens per sec: 144459.15798475762, Loss: 9.096203804016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 10.25595354508624, Tokens per sec: 144876.17360807268, Loss: 9.099984169006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 10.246429178382392, Tokens per sec: 144741.63187111693, Loss: 9.095573425292969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 10.239785869393794, Tokens per sec: 144647.78811664463, Loss: 9.09616470336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 10.244175712116268, Tokens per sec: 144709.79928055813, Loss: 9.095600128173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 10.23495538774762, Tokens per sec: 144579.55246264083, Loss: 9.09503173828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 10.238988209740013, Tokens per sec: 144636.52033174553, Loss: 9.09759521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 10.206004043414959, Tokens per sec: 144170.5841527429, Loss: 9.095932960510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 10.209967423893524, Tokens per sec: 144226.5710871379, Loss: 9.094250679016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 9.445613175989216, Tokens per sec: 133429.26021490726, Loss: 9.09061050415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 10.192843074442372, Tokens per sec: 143984.6715686676, Loss: 9.095537185668945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 10.19490921949759, Tokens per sec: 144013.85804932038, Loss: 9.087055206298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 10.229916562869468, Tokens per sec: 144508.37374048508, Loss: 9.089641571044922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 10.184498391740602, Tokens per sec: 143866.79411392962, Loss: 9.084321022033691 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 10.163606738481763, Tokens per sec: 143571.67745108771, Loss: 9.08543586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 10.11737061653748, Tokens per sec: 142918.54340554855, Loss: 9.086958885192871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 10.013086854432721, Tokens per sec: 141445.42514728277, Loss: 9.084676742553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 10.18083248893362, Tokens per sec: 143815.0093657675, Loss: 9.081693649291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 10.164748320329075, Tokens per sec: 143587.80350013543, Loss: 9.079644203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 10.206360541869653, Tokens per sec: 144175.6200698603, Loss: 9.083173751831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 10.243621527587385, Tokens per sec: 144701.97083889594, Loss: 9.078641891479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 10.258220364344554, Tokens per sec: 144908.19482374628, Loss: 9.079366683959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 10.322022508413644, Tokens per sec: 145809.46748065582, Loss: 9.079785346984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 10.273577471806602, Tokens per sec: 145125.1301829997, Loss: 9.077964782714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 9.39497490293679, Tokens per sec: 132713.94113651, Loss: 9.072598457336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 10.143726064193855, Tokens per sec: 143290.84193375445, Loss: 9.077370643615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 10.148989286481939, Tokens per sec: 143365.19050627813, Loss: 9.073976516723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 10.238245416201766, Tokens per sec: 144626.02758866423, Loss: 9.078883171081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 10.238606200290311, Tokens per sec: 144631.12404488525, Loss: 9.077786445617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 10.314438490315807, Tokens per sec: 145702.3352166729, Loss: 9.073225021362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 10.188032913565142, Tokens per sec: 143916.72296698287, Loss: 9.074674606323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 10.225046281642971, Tokens per sec: 144439.57587538278, Loss: 9.072184562683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 10.147894352257733, Tokens per sec: 143349.72340416437, Loss: 9.069905281066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 10.247179268053337, Tokens per sec: 144752.2276797769, Loss: 9.069396018981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 10.250234962523924, Tokens per sec: 144795.39259083738, Loss: 9.068591117858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 10.237310185827086, Tokens per sec: 144612.81647206453, Loss: 9.069926261901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 10.312731190020095, Tokens per sec: 145678.21779717063, Loss: 9.067359924316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 10.217898350775473, Tokens per sec: 144338.6038040151, Loss: 9.065370559692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 10.276412974583844, Tokens per sec: 145165.18465388005, Loss: 9.058939933776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 9.448145882847498, Tokens per sec: 133465.03737369468, Loss: 9.063530921936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 10.260503631823454, Tokens per sec: 144940.44838790418, Loss: 9.066680908203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 10.178410526593817, Tokens per sec: 143780.79659024437, Loss: 9.063400268554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 10.17538099843296, Tokens per sec: 143738.0013059392, Loss: 9.058425903320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 10.219634328498154, Tokens per sec: 144363.1263224553, Loss: 9.055045127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 10.260649079704024, Tokens per sec: 144942.50299280306, Loss: 9.060946464538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 10.420908944020578, Tokens per sec: 147206.34280282786, Loss: 9.060708045959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 10.435790132969052, Tokens per sec: 147416.55531053044, Loss: 9.0606050491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 10.341361876794368, Tokens per sec: 146082.6564804577, Loss: 9.05844497680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 10.377623090021936, Tokens per sec: 146594.88440735906, Loss: 9.056838035583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 10.392644089005392, Tokens per sec: 146807.0718794383, Loss: 9.05605697631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 10.433249032702685, Tokens per sec: 147380.6595859908, Loss: 9.053571701049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 10.50040061219439, Tokens per sec: 148329.24655508425, Loss: 9.04388427734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 10.495864882277852, Tokens per sec: 148265.17458050488, Loss: 9.05146598815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 10.503513816501169, Tokens per sec: 148373.2238533085, Loss: 9.045427322387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 9.670914336987098, Tokens per sec: 136611.87702097383, Loss: 9.053767204284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 10.433364452084035, Tokens per sec: 147382.29000662957, Loss: 9.04921817779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 10.459469374513642, Tokens per sec: 147751.049601465, Loss: 9.044424057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 10.448433997723026, Tokens per sec: 147595.1632514811, Loss: 9.04468059539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 10.392096873998282, Tokens per sec: 146799.34188963068, Loss: 9.044428825378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 10.513718774439328, Tokens per sec: 148517.37966011983, Loss: 9.039432525634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 10.509031081347423, Tokens per sec: 148451.1610452228, Loss: 9.042593955993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 10.505328636298348, Tokens per sec: 148398.8600993043, Loss: 9.037633895874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 10.416913604062476, Tokens per sec: 147149.9044070367, Loss: 9.038261413574219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 10.43265638944506, Tokens per sec: 147372.28787417468, Loss: 9.03659439086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 10.39248412202864, Tokens per sec: 146804.81217697437, Loss: 9.03396224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 10.449276642858758, Tokens per sec: 147607.06650381524, Loss: 9.033684730529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 10.385754220349412, Tokens per sec: 146709.7452093093, Loss: 9.034978866577148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 10.36816010214103, Tokens per sec: 146461.20971109037, Loss: 9.029942512512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 10.304219358713622, Tokens per sec: 145557.97919189048, Loss: 9.041873931884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 9.506058538948828, Tokens per sec: 134283.11479404927, Loss: 9.02549934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 10.329490219829546, Tokens per sec: 145914.95678994228, Loss: 9.029672622680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 10.302666803104426, Tokens per sec: 145536.047704488, Loss: 9.025306701660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 10.379449822586595, Tokens per sec: 146620.68893377486, Loss: 9.024807929992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 10.389260608570297, Tokens per sec: 146759.27664550324, Loss: 9.027379989624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 10.333113178013184, Tokens per sec: 145966.13489995227, Loss: 9.021807670593262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 10.336081021490617, Tokens per sec: 146008.0588229681, Loss: 9.027010917663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 10.318118285570774, Tokens per sec: 145754.31621033253, Loss: 9.021636962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 10.38729999096397, Tokens per sec: 146731.58085149789, Loss: 9.020509719848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 10.401274524889986, Tokens per sec: 146928.98589962503, Loss: 9.020039558410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 10.35326832454216, Tokens per sec: 146250.84762751986, Loss: 9.016544342041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 10.470180978110523, Tokens per sec: 147902.3623131988, Loss: 9.01088809967041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 10.484033634029814, Tokens per sec: 148098.0457057837, Loss: 9.014884948730469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 10.410669929199761, Tokens per sec: 147061.70590658655, Loss: 9.019832611083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 10.387937578741537, Tokens per sec: 146740.5874521174, Loss: 9.007903099060059 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 9.538787373147896, Tokens per sec: 134745.44413715418, Loss: 9.015063285827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 10.376350041704084, Tokens per sec: 146576.90125559285, Loss: 9.008122444152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 10.422697356833755, Tokens per sec: 147231.60602229016, Loss: 9.012078285217285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 10.362275667693353, Tokens per sec: 146378.0858608429, Loss: 9.014922142028809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 10.358733719109871, Tokens per sec: 146328.05209697728, Loss: 9.010279655456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 10.291159208640016, Tokens per sec: 145373.49078122308, Loss: 9.002447128295898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 10.335547385408743, Tokens per sec: 146000.52064981812, Loss: 9.003422737121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 10.296731317470169, Tokens per sec: 145452.2027023225, Loss: 9.001786231994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 10.255071003324534, Tokens per sec: 144863.7067737678, Loss: 9.004884719848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 10.27139684194837, Tokens per sec: 145094.32648361402, Loss: 9.001659393310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 10.385137735561193, Tokens per sec: 146701.03671069804, Loss: 9.002532958984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 10.309378115512745, Tokens per sec: 145630.85207907215, Loss: 8.999825477600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 10.387944960835302, Tokens per sec: 146740.691731987, Loss: 8.995796203613281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 10.282636779106127, Tokens per sec: 145253.1024647895, Loss: 8.991975784301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 10.298678125013824, Tokens per sec: 145479.7034146099, Loss: 8.996841430664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 9.119363690440407, Tokens per sec: 128820.64172808122, Loss: 8.997970581054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 10.358783506444126, Tokens per sec: 146328.75539564664, Loss: 8.992622375488281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 10.271228157693908, Tokens per sec: 145091.94364039745, Loss: 8.990408897399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 10.370299116616513, Tokens per sec: 146491.42554925088, Loss: 8.99206829071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 10.269082354106674, Tokens per sec: 145061.6318988645, Loss: 8.990266799926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 10.287959719684466, Tokens per sec: 145328.2946212226, Loss: 8.98782730102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 10.33715907012089, Tokens per sec: 146023.28739823718, Loss: 8.985776901245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 10.365557066988556, Tokens per sec: 146424.43909088426, Loss: 8.98536205291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 10.402290784826857, Tokens per sec: 146943.34164435393, Loss: 8.984265327453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 10.393359139889146, Tokens per sec: 146817.17272822995, Loss: 8.9846773147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 10.387923761717452, Tokens per sec: 146740.39227206213, Loss: 8.98507308959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 10.382434032210456, Tokens per sec: 146662.84404589082, Loss: 8.976456642150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 10.41224621557354, Tokens per sec: 147083.97261609725, Loss: 8.984527587890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 10.390544222066739, Tokens per sec: 146777.40904157335, Loss: 8.981645584106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 9.60703154229115, Tokens per sec: 135709.4650887968, Loss: 8.973238945007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 10.373610018918662, Tokens per sec: 146538.19554041914, Loss: 8.973687171936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 10.378619967707166, Tokens per sec: 146608.96635731688, Loss: 8.981636047363281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 10.405466189929083, Tokens per sec: 146988.1975944949, Loss: 8.972352981567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 10.427310826886949, Tokens per sec: 147296.77615838952, Loss: 8.97230339050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 10.47105682609606, Tokens per sec: 147914.734590847, Loss: 8.98348617553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 10.430293754095672, Tokens per sec: 147338.91315503753, Loss: 8.964913368225098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 10.489703140118188, Tokens per sec: 148178.13346600227, Loss: 8.965452194213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 10.484990469745629, Tokens per sec: 148111.56202064117, Loss: 8.964247703552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 10.42420712969657, Tokens per sec: 147252.93315822352, Loss: 8.96481990814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 10.435505194107744, Tokens per sec: 147412.530248233, Loss: 8.963163375854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 10.39946211211548, Tokens per sec: 146903.38365536617, Loss: 8.959172248840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 10.3661200670398, Tokens per sec: 146432.39206110852, Loss: 8.961380958557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 10.371142866849835, Tokens per sec: 146503.34441225542, Loss: 8.965780258178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 10.36603088353036, Tokens per sec: 146431.13224986425, Loss: 8.958036422729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 9.584773345023976, Tokens per sec: 135395.04454883374, Loss: 8.956931114196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 10.361894575790457, Tokens per sec: 146372.70253530014, Loss: 8.960959434509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 10.350233025928995, Tokens per sec: 146207.9708294825, Loss: 8.955310821533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 10.443756882408136, Tokens per sec: 147529.094058854, Loss: 8.960336685180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 10.336899536032766, Tokens per sec: 146019.62120518714, Loss: 8.955965995788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 10.393326438972485, Tokens per sec: 146816.71079325158, Loss: 8.95971393585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 10.433704823917676, Tokens per sec: 147387.09811819572, Loss: 8.954105377197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 10.349368784325614, Tokens per sec: 146195.76250423802, Loss: 8.945501327514648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 10.363229712270499, Tokens per sec: 146391.56274791932, Loss: 8.946640014648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 10.328700243875572, Tokens per sec: 145903.79754541654, Loss: 8.94626522064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 10.324264441332797, Tokens per sec: 145841.13715051004, Loss: 8.94890308380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 10.315334576590626, Tokens per sec: 145714.99338152783, Loss: 8.941034317016602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 10.36238402428853, Tokens per sec: 146379.616512169, Loss: 8.9400053024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 10.370348968920522, Tokens per sec: 146492.12976568664, Loss: 8.941903114318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 10.347853299536158, Tokens per sec: 146174.35468130937, Loss: 8.937877655029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 9.56840442091387, Tokens per sec: 135163.81621099781, Loss: 8.942081451416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 10.46729128249672, Tokens per sec: 147861.54231127704, Loss: 8.93345832824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 10.491650115455867, Tokens per sec: 148205.6365485574, Loss: 8.935933113098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 10.487542368330804, Tokens per sec: 148147.61028283645, Loss: 8.934524536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 10.320509659413322, Tokens per sec: 145788.09689103195, Loss: 8.932580947875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 9.922062419684234, Tokens per sec: 140159.60888912348, Loss: 8.933917999267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 10.314624847034088, Tokens per sec: 145704.96770210215, Loss: 8.934556007385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 10.335997925081445, Tokens per sec: 146006.8849984428, Loss: 8.943156242370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 10.349622962931894, Tokens per sec: 146199.35304545186, Loss: 8.931669235229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 10.366370889431524, Tokens per sec: 146435.93519224774, Loss: 8.931207656860352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 10.437392812670605, Tokens per sec: 147439.19485366598, Loss: 8.930949211120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 10.343694775490407, Tokens per sec: 146115.61113796575, Loss: 8.927345275878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 10.497028300410715, Tokens per sec: 148281.60909014408, Loss: 8.928239822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 10.397677160834656, Tokens per sec: 146878.16933370352, Loss: 8.930643081665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 10.28144828095086, Tokens per sec: 145236.3136733506, Loss: 8.927181243896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 9.471921951250717, Tokens per sec: 133800.89944625177, Loss: 8.918600082397461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 10.334550511460886, Tokens per sec: 145986.43875265465, Loss: 8.927188873291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 10.29899842942999, Tokens per sec: 145484.228052714, Loss: 8.924215316772461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 10.35240591843987, Tokens per sec: 146238.6652306665, Loss: 8.918631553649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 10.442661197078024, Tokens per sec: 147513.61634657622, Loss: 8.922369956970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 10.352717593239916, Tokens per sec: 146243.06796633042, Loss: 8.919357299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 10.384330874120971, Tokens per sec: 146689.63894084882, Loss: 8.919178009033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 10.490082915408992, Tokens per sec: 148183.49819300894, Loss: 8.91319751739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 10.51594235132396, Tokens per sec: 148548.79003160336, Loss: 8.911162376403809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 10.443736909659636, Tokens per sec: 147528.8119226912, Loss: 8.914046287536621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 10.307208454426375, Tokens per sec: 145600.20332517152, Loss: 8.909504890441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 10.343228236753717, Tokens per sec: 146109.02078566802, Loss: 8.915781021118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 10.405493805626792, Tokens per sec: 146988.58769538582, Loss: 8.915964126586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 10.359701001338152, Tokens per sec: 146341.7159798543, Loss: 8.91405200958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 10.354115440419708, Tokens per sec: 146262.81403380304, Loss: 8.916583061218262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 9.641181872658013, Tokens per sec: 136191.874566302, Loss: 8.90477180480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 10.330735723750418, Tokens per sec: 145932.55084801567, Loss: 8.900876998901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 9.784942017921873, Tokens per sec: 138222.63842181375, Loss: 8.900782585144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 10.18945318079971, Tokens per sec: 143936.78574140364, Loss: 8.903944969177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 10.351396440428148, Tokens per sec: 146224.40528779165, Loss: 8.9007568359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 10.321118779089009, Tokens per sec: 145796.70134965153, Loss: 8.898226737976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 10.348007758221629, Tokens per sec: 146176.53657334243, Loss: 8.895816802978516 +------------------------------------------------------------------ +[rank0]:[2024-08-02 17:53:39,031] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010386829999333713, 'preprocessing_with_comm': 0.0004994969999643217, 'state_converting': 0.027937375999954384, : 0.029787773999942146}) +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 10.361870129060945, Tokens per sec: 146372.35719943122, Loss: 8.9003324508667 +------------------------------------------------------------------ + eval ppl=7357.109375, eval loss=8.903422355651855 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001000, took 0.16s +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 10.302231877652819, Tokens per sec: 145529.90392322413, Loss: 8.90183162689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 10.305561414185235, Tokens per sec: 145576.93714257173, Loss: 8.901196479797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 10.308464309157603, Tokens per sec: 145617.94359937106, Loss: 8.904306411743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 10.266326379663694, Tokens per sec: 145022.7008496836, Loss: 8.89440631866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 10.319730825364783, Tokens per sec: 145777.09503768582, Loss: 8.892809867858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 10.343281898164264, Tokens per sec: 146109.77880975584, Loss: 8.89244556427002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 9.53688789754688, Tokens per sec: 134718.61203853678, Loss: 8.889389038085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 10.246332610738085, Tokens per sec: 144740.26775117038, Loss: 8.891561508178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 10.295575815969197, Tokens per sec: 145435.88002346843, Loss: 8.897067070007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 10.364279952117894, Tokens per sec: 146406.3984947655, Loss: 8.893034934997559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 10.385192491642714, Tokens per sec: 146701.8101981693, Loss: 8.88961124420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 10.324154981884432, Tokens per sec: 145839.59092021806, Loss: 8.886089324951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 10.303970283228333, Tokens per sec: 145554.46073764926, Loss: 8.892110824584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 10.307741499523672, Tokens per sec: 145607.7331500403, Loss: 8.89091968536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 10.31534654389255, Tokens per sec: 145715.1624323044, Loss: 8.88542366027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 10.286205980446379, Tokens per sec: 145303.52120262204, Loss: 8.89043140411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 10.23604292704963, Tokens per sec: 144594.91510366983, Loss: 8.886187553405762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 10.254180207781902, Tokens per sec: 144851.1233460917, Loss: 8.891559600830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 10.18418733841887, Tokens per sec: 143862.40015730308, Loss: 8.87689208984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 10.246528846705047, Tokens per sec: 144743.0397914193, Loss: 8.886117935180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 10.223370845381675, Tokens per sec: 144415.9085690135, Loss: 8.88212776184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 9.463545405067217, Tokens per sec: 133682.57188618806, Loss: 8.885056495666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 10.273443182318168, Tokens per sec: 145123.23320217244, Loss: 8.874637603759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 10.248898148226889, Tokens per sec: 144776.50867728147, Loss: 8.877886772155762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 10.268863766071773, Tokens per sec: 145058.54411205326, Loss: 8.883862495422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 10.217027960140511, Tokens per sec: 144326.30861720646, Loss: 8.870983123779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 10.223400542899475, Tokens per sec: 144416.32807781157, Loss: 8.871609687805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 10.287852431264067, Tokens per sec: 145326.77905899315, Loss: 8.87610912322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 10.299396880005933, Tokens per sec: 145489.85658784345, Loss: 8.873397827148438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 10.305368156170841, Tokens per sec: 145574.20716904744, Loss: 8.878289222717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 10.23782239752211, Tokens per sec: 144620.05200312714, Loss: 8.876431465148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 10.373943928613821, Tokens per sec: 146542.91236745552, Loss: 8.878830909729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 10.277937310794098, Tokens per sec: 145186.71751247413, Loss: 8.875224113464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 10.259237852214646, Tokens per sec: 144922.56791432863, Loss: 8.869434356689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 10.28320031620752, Tokens per sec: 145261.0630214142, Loss: 8.87241268157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 10.282972337151138, Tokens per sec: 145257.84257650812, Loss: 8.866511344909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 10.297477594289578, Tokens per sec: 145462.74465042842, Loss: 8.871753692626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 9.434487102983113, Tokens per sec: 133272.09268510886, Loss: 8.872153282165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 10.212419937241506, Tokens per sec: 144261.21542791193, Loss: 8.866555213928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 10.351133646225637, Tokens per sec: 146220.6930421834, Loss: 8.864462852478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 10.311292169705535, Tokens per sec: 145657.89011569263, Loss: 8.870817184448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 10.226971213032373, Tokens per sec: 144466.7675638912, Loss: 8.867571830749512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 10.284366420402058, Tokens per sec: 145277.5354745109, Loss: 8.86430549621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 10.26965539673487, Tokens per sec: 145069.72673109264, Loss: 8.865838050842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 10.284503180000167, Tokens per sec: 145279.46734824555, Loss: 8.863529205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 10.2942479773172, Tokens per sec: 145417.12290037647, Loss: 8.862478256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 10.293272720449842, Tokens per sec: 145403.34636730168, Loss: 8.860126495361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 10.28477628231245, Tokens per sec: 145283.3252067892, Loss: 8.858552932739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 10.283352277526724, Tokens per sec: 145263.20963551165, Loss: 8.860551834106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 10.311396315297124, Tokens per sec: 145659.36128214642, Loss: 8.862183570861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 10.260881632754938, Tokens per sec: 144945.78805021182, Loss: 8.860176086425781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 10.315259074257247, Tokens per sec: 145713.9268313421, Loss: 8.85264778137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 9.513964334778013, Tokens per sec: 134394.79250826902, Loss: 8.853257179260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 10.263732938111144, Tokens per sec: 144986.06574920678, Loss: 8.861425399780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 10.207288458139232, Tokens per sec: 144188.72786700216, Loss: 8.848825454711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 10.293098959051177, Tokens per sec: 145400.891804062, Loss: 8.858179092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 10.323332330933852, Tokens per sec: 145827.97010686208, Loss: 8.849493980407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 10.275949177791194, Tokens per sec: 145158.6330344372, Loss: 8.852461814880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 10.388602594462562, Tokens per sec: 146749.98150140088, Loss: 8.852686882019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 10.43468054657254, Tokens per sec: 147400.88123101092, Loss: 8.850874900817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 10.491250194912238, Tokens per sec: 148199.98724858215, Loss: 8.858994483947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 10.505933335693243, Tokens per sec: 148407.40211679006, Loss: 8.854461669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 10.468701692992125, Tokens per sec: 147881.4658488489, Loss: 8.848860740661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 10.46432984394746, Tokens per sec: 147819.70886463491, Loss: 8.858820915222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 10.405278921356361, Tokens per sec: 146985.55222815878, Loss: 8.858945846557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 10.338200638515897, Tokens per sec: 146038.00065166177, Loss: 8.85135269165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 9.887121707525854, Tokens per sec: 139666.0344342087, Loss: 8.852237701416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 9.63530733710332, Tokens per sec: 136108.8905483707, Loss: 8.858699798583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 10.392871835756209, Tokens per sec: 146810.28904278297, Loss: 8.844977378845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 10.445972126101813, Tokens per sec: 147560.38671521586, Loss: 8.850334167480469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 10.450546833104987, Tokens per sec: 147625.0092823018, Loss: 8.845410346984863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 10.392860719578838, Tokens per sec: 146810.13201503947, Loss: 8.84616756439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 10.385180138990167, Tokens per sec: 146701.63570390825, Loss: 8.84734058380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 10.369135168788322, Tokens per sec: 146474.9835571059, Loss: 8.843366622924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 10.333195989162348, Tokens per sec: 145967.30469487872, Loss: 8.84700870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 10.305073398430194, Tokens per sec: 145570.0434047111, Loss: 8.847371101379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 10.370434793363206, Tokens per sec: 146493.34212656593, Loss: 8.846813201904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 10.323281911730522, Tokens per sec: 145827.25788237483, Loss: 8.851127624511719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 10.349117926885038, Tokens per sec: 146192.21887799675, Loss: 8.840577125549316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 10.360981384988293, Tokens per sec: 146359.8027509349, Loss: 8.84292221069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 10.37370659289506, Tokens per sec: 146539.55974981314, Loss: 8.849233627319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 10.374937765416117, Tokens per sec: 146556.95136173084, Loss: 8.847707748413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 9.571401611639002, Tokens per sec: 135206.15469487666, Loss: 8.849262237548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 10.382389276615424, Tokens per sec: 146662.21182585126, Loss: 8.846110343933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 10.361203625995959, Tokens per sec: 146362.94213984377, Loss: 8.83859920501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 10.368041754846622, Tokens per sec: 146459.53793058792, Loss: 8.845995903015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 10.382556186815245, Tokens per sec: 146664.5696086727, Loss: 8.838829040527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 10.378012707749765, Tokens per sec: 146600.38816918185, Loss: 8.841133117675781 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 10.41709925600978, Tokens per sec: 147152.52693683174, Loss: 8.836677551269531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 10.345897040751828, Tokens per sec: 146146.72046026742, Loss: 8.83343505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 10.398691340770156, Tokens per sec: 146892.49569621673, Loss: 8.845216751098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 10.358702472998294, Tokens per sec: 146327.6107126569, Loss: 8.838750839233398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 10.41269267223171, Tokens per sec: 147090.27928783023, Loss: 8.842296600341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 10.391315904587396, Tokens per sec: 146788.30987203654, Loss: 8.838808059692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 10.31604991668506, Tokens per sec: 145725.09831572577, Loss: 8.846268653869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 10.370822538783061, Tokens per sec: 146498.81944006152, Loss: 8.830788612365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 9.511406397321439, Tokens per sec: 134358.65894063844, Loss: 8.839399337768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 10.498567396840775, Tokens per sec: 148303.35045242897, Loss: 8.84593391418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 10.452721258617347, Tokens per sec: 147655.72533875058, Loss: 8.83621597290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 10.36643669243964, Tokens per sec: 146436.86472922214, Loss: 8.844399452209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 10.320684998289064, Tokens per sec: 145790.57373780105, Loss: 8.828898429870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 10.268556502697361, Tokens per sec: 145054.20369243465, Loss: 8.828425407409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 10.453571121361731, Tokens per sec: 147667.73054742833, Loss: 8.83685302734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 10.262273656268064, Tokens per sec: 144965.45185224328, Loss: 8.843866348266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 10.189638747373797, Tokens per sec: 143939.40706521182, Loss: 8.847246170043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 10.241318058658907, Tokens per sec: 144669.43190793105, Loss: 8.838722229003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 10.268414178557812, Tokens per sec: 145052.1932136762, Loss: 8.83642864227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 10.259861590530543, Tokens per sec: 144931.3788766777, Loss: 8.831746101379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 10.264032814628697, Tokens per sec: 144990.30182167215, Loss: 8.835882186889648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 10.281714243850242, Tokens per sec: 145240.07068014814, Loss: 8.840608596801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 10.215373953802729, Tokens per sec: 144302.94403113573, Loss: 8.834030151367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 10.229559038285057, Tokens per sec: 144503.32332820183, Loss: 8.837454795837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 9.4888106521753, Tokens per sec: 134039.47018045103, Loss: 8.827889442443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 10.203759308228767, Tokens per sec: 144138.8748979077, Loss: 8.829245567321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 10.236345542700032, Tokens per sec: 144599.18986927907, Loss: 8.832292556762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 10.307343564095389, Tokens per sec: 145602.1118919155, Loss: 8.829325675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 10.38652704814452, Tokens per sec: 146720.66221798348, Loss: 8.829216003417969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 10.432393779931342, Tokens per sec: 147368.5782374906, Loss: 8.831982612609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 10.401980224111316, Tokens per sec: 146938.95464631004, Loss: 8.835164070129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 10.314616332848322, Tokens per sec: 145704.84743023766, Loss: 8.834166526794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 10.33475589452355, Tokens per sec: 145989.3400052892, Loss: 8.826164245605469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 10.274917988790358, Tokens per sec: 145144.0664009154, Loss: 8.836819648742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 9.873892345313603, Tokens per sec: 139479.15572340327, Loss: 8.832469940185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 10.353140973496902, Tokens per sec: 146249.04865952913, Loss: 8.831230163574219 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 10.386624308374001, Tokens per sec: 146722.0361214269, Loss: 8.829593658447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 10.444819593580322, Tokens per sec: 147544.105976332, Loss: 8.827186584472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 10.378894346579221, Tokens per sec: 146612.84224861526, Loss: 8.824728012084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 10.375271315965538, Tokens per sec: 146561.6631154544, Loss: 8.822948455810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 9.503532859900362, Tokens per sec: 134247.43691049656, Loss: 8.826034545898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 10.432242579411882, Tokens per sec: 147366.44237049288, Loss: 8.830243110656738 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 10.448221676197587, Tokens per sec: 147592.16398573318, Loss: 8.830349922180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 10.44001307363771, Tokens per sec: 147476.20880669335, Loss: 8.823662757873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 10.421594010709711, Tokens per sec: 147216.0200932087, Loss: 8.826601028442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 10.38386992240367, Tokens per sec: 146683.12751109956, Loss: 8.835984230041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 10.387585023918167, Tokens per sec: 146735.60724295673, Loss: 8.827674865722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 10.431547441213224, Tokens per sec: 147356.62280940503, Loss: 8.824167251586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 10.390375512792234, Tokens per sec: 146775.0258449223, Loss: 8.819933891296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 10.402690526124223, Tokens per sec: 146948.98841228645, Loss: 8.826179504394531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 10.395903014854705, Tokens per sec: 146853.10764832562, Loss: 8.825345993041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 10.409507410336762, Tokens per sec: 147045.2841000838, Loss: 8.823514938354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 10.403732974021459, Tokens per sec: 146963.71408960875, Loss: 8.822980880737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 10.37021144362065, Tokens per sec: 146490.18707560597, Loss: 8.822589874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 10.342964623598006, Tokens per sec: 146105.29697148103, Loss: 8.82751750946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 9.592775729488212, Tokens per sec: 135508.08667951645, Loss: 8.827533721923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 10.404413073518896, Tokens per sec: 146973.32121316175, Loss: 8.831341743469238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 10.334450400867441, Tokens per sec: 145985.0245848104, Loss: 8.822457313537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 10.367404618445253, Tokens per sec: 146450.5377061338, Loss: 8.831897735595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 10.327511885772685, Tokens per sec: 145887.01073236545, Loss: 8.824125289916992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 10.374451436942786, Tokens per sec: 146550.08145850606, Loss: 8.820399284362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 10.420839055571431, Tokens per sec: 147205.3555546849, Loss: 8.826271057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 10.40176317595687, Tokens per sec: 146935.8886119363, Loss: 8.8318452835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 10.524631739538718, Tokens per sec: 148671.53681570443, Loss: 8.815516471862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 10.551483499622499, Tokens per sec: 149050.846281029, Loss: 8.820700645446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 10.555331760988397, Tokens per sec: 149105.20703639748, Loss: 8.820707321166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 10.545871806741573, Tokens per sec: 148971.57519341356, Loss: 8.826807975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 10.53924663575324, Tokens per sec: 148877.98765734787, Loss: 8.826924324035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 10.554329511142502, Tokens per sec: 149091.04919899756, Loss: 8.817045211791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 10.553376366947688, Tokens per sec: 149077.58503077232, Loss: 8.817075729370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 9.74426841255076, Tokens per sec: 137648.0807966152, Loss: 8.819220542907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 10.382179946114153, Tokens per sec: 146659.25481147805, Loss: 8.827580451965332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 10.39637062528059, Tokens per sec: 146859.7131393649, Loss: 8.818222999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 10.412616913524825, Tokens per sec: 147089.20911609803, Loss: 8.815677642822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 10.387807507008853, Tokens per sec: 146738.7500515439, Loss: 8.812749862670898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 10.354619672684326, Tokens per sec: 146269.93684698528, Loss: 8.823498725891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 10.365828813904603, Tokens per sec: 146428.27780302483, Loss: 8.830415725708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 10.466441610126651, Tokens per sec: 147849.53979183748, Loss: 8.814091682434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 10.532303287657339, Tokens per sec: 148779.90553365834, Loss: 8.817205429077148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1168 , TFLOPS: 10.511949425320884, Tokens per sec: 148492.38573547598, Loss: 8.821809768676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1169 , TFLOPS: 10.525401732737595, Tokens per sec: 148682.41378271361, Loss: 8.818521499633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1170 , TFLOPS: 10.408529145965279, Tokens per sec: 147031.46508283744, Loss: 8.820075035095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1171 , TFLOPS: 10.360571153754748, Tokens per sec: 146354.00780157698, Loss: 8.818137168884277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1172 , TFLOPS: 10.388669209371436, Tokens per sec: 146750.9225073308, Loss: 8.810420989990234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1173 , TFLOPS: 9.582841091234037, Tokens per sec: 135367.74942368572, Loss: 8.819741249084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1174 , TFLOPS: 10.363141992423321, Tokens per sec: 146390.32361245007, Loss: 8.81538200378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1175 , TFLOPS: 10.392308072962738, Tokens per sec: 146802.32529801934, Loss: 8.819273948669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1176 , TFLOPS: 10.426500289827416, Tokens per sec: 147285.32646653627, Loss: 8.816685676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1177 , TFLOPS: 10.432353528042992, Tokens per sec: 147368.00963706363, Loss: 8.817669868469238 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1178 , TFLOPS: 10.506330311097514, Tokens per sec: 148413.00981356198, Loss: 8.818995475769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1179 , TFLOPS: 10.465074216621291, Tokens per sec: 147830.2239146738, Loss: 8.817801475524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1180 , TFLOPS: 10.331921241471017, Tokens per sec: 145949.29753766765, Loss: 8.822150230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1181 , TFLOPS: 10.35090708891441, Tokens per sec: 146217.4926809289, Loss: 8.808029174804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1182 , TFLOPS: 10.404971228569972, Tokens per sec: 146981.20574264252, Loss: 8.809823036193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1183 , TFLOPS: 10.275944143862406, Tokens per sec: 145158.5619248775, Loss: 8.81257438659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1184 , TFLOPS: 10.393447799629195, Tokens per sec: 146818.4251406785, Loss: 8.82149600982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1185 , TFLOPS: 10.345572698180757, Tokens per sec: 146142.1387789612, Loss: 8.817938804626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1186 , TFLOPS: 10.377108080688423, Tokens per sec: 146587.6093566986, Loss: 8.82287883758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1187 , TFLOPS: 10.306316749955089, Tokens per sec: 145587.60705791842, Loss: 8.808341026306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1188 , TFLOPS: 9.543427903952635, Tokens per sec: 134810.99653494396, Loss: 8.811714172363281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1189 , TFLOPS: 10.325509593531903, Tokens per sec: 145858.72624014204, Loss: 8.809576988220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1190 , TFLOPS: 10.344583337195859, Tokens per sec: 146128.16301033285, Loss: 8.812750816345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1191 , TFLOPS: 10.3910522131353, Tokens per sec: 146784.58495183045, Loss: 8.820260047912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1192 , TFLOPS: 10.398459979983432, Tokens per sec: 146889.2274807986, Loss: 8.816546440124512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1193 , TFLOPS: 10.41162031336187, Tokens per sec: 147075.13108643537, Loss: 8.810190200805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1194 , TFLOPS: 10.440989029097178, Tokens per sec: 147489.99520811945, Loss: 8.816032409667969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1195 , TFLOPS: 10.360663457801724, Tokens per sec: 146355.31169370908, Loss: 8.811910629272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1196 , TFLOPS: 10.423744809613764, Tokens per sec: 147246.4023988665, Loss: 8.802183151245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1197 , TFLOPS: 10.489127810406812, Tokens per sec: 148170.0063263091, Loss: 8.81102466583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1198 , TFLOPS: 10.46922995572957, Tokens per sec: 147888.92811783485, Loss: 8.80966854095459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1199 , TFLOPS: 10.472124845870393, Tokens per sec: 147929.82149793595, Loss: 8.819371223449707 +------------------------------------------------------------------ +[rank0]:[2024-08-02 17:57:25,413] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010081769996759249, 'preprocessing_with_comm': 0.0004975659999217896, 'state_converting': 0.028850319999946805, : 0.030669776999729947}) +------------------------------------------------------------------ +iteration: 1200 , TFLOPS: 10.407608779827257, Tokens per sec: 147018.4639392764, Loss: 8.814531326293945 +------------------------------------------------------------------ + eval ppl=6767.3330078125, eval loss=8.819862365722656 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001200, took 0.17s +------------------------------------------------------------------ +iteration: 1201 , TFLOPS: 10.423962102892412, Tokens per sec: 147249.47189587847, Loss: 8.801531791687012 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1202 , TFLOPS: 9.629717693742377, Tokens per sec: 136029.9309335082, Loss: 8.817606925964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1203 , TFLOPS: 10.385931931690127, Tokens per sec: 146712.2555696514, Loss: 8.812582969665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1204 , TFLOPS: 10.412689770616629, Tokens per sec: 147090.23829945322, Loss: 8.812396049499512 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1205 , TFLOPS: 10.411024966837008, Tokens per sec: 147066.72118811499, Loss: 8.812519073486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1206 , TFLOPS: 10.45442041695524, Tokens per sec: 147679.72774450114, Loss: 8.815546035766602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1207 , TFLOPS: 10.42824985012474, Tokens per sec: 147310.04085318578, Loss: 8.81312370300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1208 , TFLOPS: 10.37764882515107, Tokens per sec: 146595.24794323312, Loss: 8.816675186157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1209 , TFLOPS: 10.4017697584035, Tokens per sec: 146935.9815959457, Loss: 8.810173034667969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1210 , TFLOPS: 10.364871850203164, Tokens per sec: 146414.75968023532, Loss: 8.816495895385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1211 , TFLOPS: 10.466309593288175, Tokens per sec: 147847.6749145907, Loss: 8.808027267456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1212 , TFLOPS: 10.420574209374703, Tokens per sec: 147201.6143224915, Loss: 8.809651374816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1213 , TFLOPS: 10.435912384254022, Tokens per sec: 147418.28223902202, Loss: 8.81026840209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1214 , TFLOPS: 10.386870405233338, Tokens per sec: 146725.51249943126, Loss: 8.80860424041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1215 , TFLOPS: 10.425884056137198, Tokens per sec: 147276.62151494942, Loss: 8.806520462036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1216 , TFLOPS: 10.445954933603968, Tokens per sec: 147560.14385302932, Loss: 8.813821792602539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1217 , TFLOPS: 9.633022689969303, Tokens per sec: 136076.61749512743, Loss: 8.811649322509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1218 , TFLOPS: 10.393893742594239, Tokens per sec: 146824.7245559536, Loss: 8.802563667297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1219 , TFLOPS: 10.389187753901444, Tokens per sec: 146758.24749637474, Loss: 8.804841995239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1220 , TFLOPS: 10.40593185640159, Tokens per sec: 146994.77562513997, Loss: 8.807168960571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1221 , TFLOPS: 10.453804903457694, Tokens per sec: 147671.03296639625, Loss: 8.809550285339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1222 , TFLOPS: 10.480613373282797, Tokens per sec: 148049.7309111046, Loss: 8.801682472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1223 , TFLOPS: 10.473844884632927, Tokens per sec: 147954.11886173324, Loss: 8.8080415725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1224 , TFLOPS: 10.445295417349177, Tokens per sec: 147550.8274895136, Loss: 8.80459976196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1225 , TFLOPS: 10.445176973694752, Tokens per sec: 147549.15434782417, Loss: 8.813093185424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1226 , TFLOPS: 10.470620548897585, Tokens per sec: 147908.57171473122, Loss: 8.801740646362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1227 , TFLOPS: 10.441402651458032, Tokens per sec: 147495.83806073142, Loss: 8.817225456237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1228 , TFLOPS: 10.425679141234815, Tokens per sec: 147273.72687557316, Loss: 8.808821678161621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1229 , TFLOPS: 10.48654824701441, Tokens per sec: 148133.56726949906, Loss: 8.814170837402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1230 , TFLOPS: 10.435024705444217, Tokens per sec: 147405.74283848822, Loss: 8.8013334274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1231 , TFLOPS: 10.402819965120475, Tokens per sec: 146950.81687478974, Loss: 8.804201126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1232 , TFLOPS: 9.633792671993106, Tokens per sec: 136087.49430427686, Loss: 8.809566497802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1233 , TFLOPS: 10.404008123472627, Tokens per sec: 146967.6008661508, Loss: 8.810043334960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1234 , TFLOPS: 10.429543547261266, Tokens per sec: 147328.31569131993, Loss: 8.807599067687988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1235 , TFLOPS: 10.431700775313182, Tokens per sec: 147358.78881548025, Loss: 8.809782028198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1236 , TFLOPS: 10.456716926542962, Tokens per sec: 147712.16836742914, Loss: 8.80208683013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1237 , TFLOPS: 10.429976173988074, Tokens per sec: 147334.4270006687, Loss: 8.801819801330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1238 , TFLOPS: 10.43736124021322, Tokens per sec: 147438.74885936646, Loss: 8.802759170532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1239 , TFLOPS: 10.374718834158205, Tokens per sec: 146553.8587265322, Loss: 8.800548553466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1240 , TFLOPS: 10.491104765632373, Tokens per sec: 148197.93290643787, Loss: 8.80265998840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1241 , TFLOPS: 10.383478352507058, Tokens per sec: 146677.59617283128, Loss: 8.808155059814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1242 , TFLOPS: 10.463179842207268, Tokens per sec: 147803.4638757093, Loss: 8.80341625213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1243 , TFLOPS: 10.4298222259731, Tokens per sec: 147332.25232239565, Loss: 8.803079605102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1244 , TFLOPS: 10.428832119179598, Tokens per sec: 147318.26601843323, Loss: 8.815567016601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1245 , TFLOPS: 10.447850655599865, Tokens per sec: 147586.9229280108, Loss: 8.80733585357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1246 , TFLOPS: 10.457367562955946, Tokens per sec: 147721.3592938027, Loss: 8.802434921264648 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1247 , TFLOPS: 9.587517934920022, Tokens per sec: 135433.81477926768, Loss: 8.805668830871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1248 , TFLOPS: 10.418459202811526, Tokens per sec: 147171.7376214435, Loss: 8.803912162780762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1249 , TFLOPS: 10.421846478057475, Tokens per sec: 147219.58646108897, Loss: 8.797323226928711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1250 , TFLOPS: 10.417351216138616, Tokens per sec: 147156.08613970905, Loss: 8.80462646484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1251 , TFLOPS: 10.407225007998417, Tokens per sec: 147013.0427569498, Loss: 8.804065704345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1252 , TFLOPS: 10.376881682171812, Tokens per sec: 146584.41123858574, Loss: 8.800387382507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1253 , TFLOPS: 10.452572505006197, Tokens per sec: 147653.62403691653, Loss: 8.804473876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1254 , TFLOPS: 10.384886867259038, Tokens per sec: 146697.49293102548, Loss: 8.796370506286621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1255 , TFLOPS: 10.423486406960889, Tokens per sec: 147242.75218853416, Loss: 8.81094741821289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1256 , TFLOPS: 10.42465205975509, Tokens per sec: 147259.21826512442, Loss: 8.80512523651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1257 , TFLOPS: 10.38700575465342, Tokens per sec: 146727.42445291224, Loss: 8.811596870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1258 , TFLOPS: 10.409242682948106, Tokens per sec: 147041.5445461799, Loss: 8.794458389282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1259 , TFLOPS: 10.395851196103859, Tokens per sec: 146852.37565375186, Loss: 8.805401802062988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1260 , TFLOPS: 10.400346105440752, Tokens per sec: 146915.87099453923, Loss: 8.801918983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1261 , TFLOPS: 10.386093804111328, Tokens per sec: 146714.54218853018, Loss: 8.801107406616211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1262 , TFLOPS: 9.596083201684921, Tokens per sec: 135554.80821682332, Loss: 8.802407264709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1263 , TFLOPS: 10.390529636299691, Tokens per sec: 146777.203002212, Loss: 8.8073148727417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1264 , TFLOPS: 10.448336393752827, Tokens per sec: 147593.7844923371, Loss: 8.793601989746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1265 , TFLOPS: 10.409398056673705, Tokens per sec: 147043.73936412102, Loss: 8.805543899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1266 , TFLOPS: 10.367016943593903, Tokens per sec: 146445.06138949288, Loss: 8.80817985534668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1267 , TFLOPS: 10.091143215182852, Tokens per sec: 142548.05366656548, Loss: 8.797861099243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1268 , TFLOPS: 10.325995197774297, Tokens per sec: 145865.5859128401, Loss: 8.796645164489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1269 , TFLOPS: 10.383639146436794, Tokens per sec: 146679.86755687936, Loss: 8.800444602966309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1270 , TFLOPS: 10.384412104889606, Tokens per sec: 146690.7864112315, Loss: 8.795924186706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1271 , TFLOPS: 10.466832373087938, Tokens per sec: 147855.05973129216, Loss: 8.81049633026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1272 , TFLOPS: 10.462597452242589, Tokens per sec: 147795.23700248296, Loss: 8.807689666748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1273 , TFLOPS: 10.385670230510467, Tokens per sec: 146708.55876414507, Loss: 8.802000999450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1274 , TFLOPS: 10.397798223076835, Tokens per sec: 146879.8794657101, Loss: 8.80543327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1275 , TFLOPS: 10.470570520342358, Tokens per sec: 147907.86500856094, Loss: 8.800414085388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1276 , TFLOPS: 10.428277221930108, Tokens per sec: 147310.4275088399, Loss: 8.792186737060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1277 , TFLOPS: 9.713503644185751, Tokens per sec: 137213.49595737355, Loss: 8.800549507141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1278 , TFLOPS: 10.481752755263493, Tokens per sec: 148065.82588471335, Loss: 8.797725677490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1279 , TFLOPS: 10.44222955350311, Tokens per sec: 147507.51892528613, Loss: 8.803218841552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1280 , TFLOPS: 10.470668757143212, Tokens per sec: 147909.25270710624, Loss: 8.792378425598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1281 , TFLOPS: 10.51102081387287, Tokens per sec: 148479.26811820467, Loss: 8.807291030883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1282 , TFLOPS: 10.426956726493971, Tokens per sec: 147291.77411642612, Loss: 8.796314239501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1283 , TFLOPS: 10.431183680530713, Tokens per sec: 147351.48430565116, Loss: 8.793274879455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1284 , TFLOPS: 10.144320533613259, Tokens per sec: 143299.23944203407, Loss: 8.78812313079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1285 , TFLOPS: 10.424864004233294, Tokens per sec: 147262.212204682, Loss: 8.80058765411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1286 , TFLOPS: 10.408571367729111, Tokens per sec: 147032.0615098357, Loss: 8.806556701660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1287 , TFLOPS: 10.457436248374352, Tokens per sec: 147722.32954786604, Loss: 8.797607421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1288 , TFLOPS: 10.463538187819553, Tokens per sec: 147808.52588587822, Loss: 8.798250198364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1289 , TFLOPS: 10.393243330492673, Tokens per sec: 146815.53679821576, Loss: 8.79382610321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1290 , TFLOPS: 10.475591211225744, Tokens per sec: 147978.787568892, Loss: 8.797727584838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1291 , TFLOPS: 9.732385109721363, Tokens per sec: 137480.21659596523, Loss: 8.792208671569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1292 , TFLOPS: 10.431759441430764, Tokens per sec: 147359.61753633962, Loss: 8.800786972045898 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1293 , TFLOPS: 10.419201603634454, Tokens per sec: 147182.22481700618, Loss: 8.792094230651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1294 , TFLOPS: 10.43248748992907, Tokens per sec: 147369.90199016168, Loss: 8.802373886108398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1295 , TFLOPS: 10.43839405302863, Tokens per sec: 147453.33843098377, Loss: 8.799330711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1296 , TFLOPS: 10.381060150896857, Tokens per sec: 146643.43652158495, Loss: 8.794380187988281 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1297 , TFLOPS: 10.385321801825139, Tokens per sec: 146703.63684104124, Loss: 8.805590629577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1298 , TFLOPS: 10.376789144518815, Tokens per sec: 146583.10404652194, Loss: 8.80038070678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1299 , TFLOPS: 10.423382864372805, Tokens per sec: 147241.2895401416, Loss: 8.799087524414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1300 , TFLOPS: 10.439762173159822, Tokens per sec: 147472.6645725046, Loss: 8.799246788024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1301 , TFLOPS: 10.388227930518703, Tokens per sec: 146744.68897756713, Loss: 8.80009937286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1302 , TFLOPS: 10.39965241007024, Tokens per sec: 146906.07181492247, Loss: 8.792402267456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1303 , TFLOPS: 10.412450238923975, Tokens per sec: 147086.85466136082, Loss: 8.801739692687988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1304 , TFLOPS: 10.448347112253574, Tokens per sec: 147593.93590247832, Loss: 8.793620109558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1305 , TFLOPS: 10.474782753270464, Tokens per sec: 147967.3672465817, Loss: 8.79592514038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1306 , TFLOPS: 9.598567201730676, Tokens per sec: 135589.8973404518, Loss: 8.804061889648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1307 , TFLOPS: 10.488465728198955, Tokens per sec: 148160.65371599683, Loss: 8.797694206237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1308 , TFLOPS: 10.477980187052028, Tokens per sec: 148012.5343750795, Loss: 8.793068885803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1309 , TFLOPS: 10.354720322052033, Tokens per sec: 146271.35862558498, Loss: 8.802539825439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1310 , TFLOPS: 10.39097872941442, Tokens per sec: 146783.5469166778, Loss: 8.796891212463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1311 , TFLOPS: 10.365793567637448, Tokens per sec: 146427.77991228295, Loss: 8.791024208068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1312 , TFLOPS: 10.435335274648141, Tokens per sec: 147410.1299564395, Loss: 8.802762031555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1313 , TFLOPS: 10.430195243684377, Tokens per sec: 147337.52159145582, Loss: 8.800902366638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1314 , TFLOPS: 10.414951385067486, Tokens per sec: 147122.18599172548, Loss: 8.789817810058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1315 , TFLOPS: 10.358399319802762, Tokens per sec: 146323.3283536551, Loss: 8.794670104980469 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1316 , TFLOPS: 10.494912951287818, Tokens per sec: 148251.72755007824, Loss: 8.7914400100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1317 , TFLOPS: 10.464368392448248, Tokens per sec: 147820.25340291386, Loss: 8.797382354736328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1318 , TFLOPS: 10.398729193172647, Tokens per sec: 146893.0304013722, Loss: 8.792729377746582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1319 , TFLOPS: 10.432261112828833, Tokens per sec: 147366.70417457772, Loss: 8.804864883422852 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1320 , TFLOPS: 9.654160055333238, Tokens per sec: 136375.20510091368, Loss: 8.79358196258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1321 , TFLOPS: 10.464369362994251, Tokens per sec: 147820.26711290103, Loss: 8.804961204528809 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1322 , TFLOPS: 10.419258382347564, Tokens per sec: 147183.02687628442, Loss: 8.794944763183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1323 , TFLOPS: 10.417914066369544, Tokens per sec: 147164.03699356318, Loss: 8.788240432739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1324 , TFLOPS: 10.32681819978346, Tokens per sec: 145877.21168526929, Loss: 8.791690826416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1325 , TFLOPS: 10.392851220656505, Tokens per sec: 146809.9978327311, Loss: 8.794626235961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1326 , TFLOPS: 10.429432209111825, Tokens per sec: 147326.74292239145, Loss: 8.793536186218262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1327 , TFLOPS: 10.400133946740217, Tokens per sec: 146912.87402886496, Loss: 8.786656379699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1328 , TFLOPS: 10.384789194120696, Tokens per sec: 146696.11319480836, Loss: 8.792459487915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1329 , TFLOPS: 10.376677113807864, Tokens per sec: 146581.5214944308, Loss: 8.793092727661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1330 , TFLOPS: 10.360813680538193, Tokens per sec: 146357.43374848957, Loss: 8.795561790466309 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1331 , TFLOPS: 10.379153713344566, Tokens per sec: 146616.50607805434, Loss: 8.795126914978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1332 , TFLOPS: 10.408474357930894, Tokens per sec: 147030.69114399827, Loss: 8.791258811950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1333 , TFLOPS: 10.413331039088256, Tokens per sec: 147099.296893763, Loss: 8.80289077758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1334 , TFLOPS: 10.411339239234206, Tokens per sec: 147071.16061758163, Loss: 8.797648429870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1335 , TFLOPS: 10.46312735060044, Tokens per sec: 147802.72237633428, Loss: 8.788923263549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1336 , TFLOPS: 9.612577125227801, Tokens per sec: 135787.8023036405, Loss: 8.79247760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1337 , TFLOPS: 10.34020258438883, Tokens per sec: 146066.28025107356, Loss: 8.790040969848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1338 , TFLOPS: 10.416766222674411, Tokens per sec: 147147.82248930278, Loss: 8.789081573486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1339 , TFLOPS: 10.377738542305613, Tokens per sec: 146596.51529277794, Loss: 8.798637390136719 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1340 , TFLOPS: 10.187709524740338, Tokens per sec: 143912.1547583498, Loss: 8.791537284851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1341 , TFLOPS: 10.479929229776145, Tokens per sec: 148040.06666165113, Loss: 8.79384708404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1342 , TFLOPS: 10.433499695149237, Tokens per sec: 147384.20045773557, Loss: 8.791585922241211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1343 , TFLOPS: 10.24964929552229, Tokens per sec: 144787.1194260037, Loss: 8.798216819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1344 , TFLOPS: 10.484729816271491, Tokens per sec: 148107.88001508167, Loss: 8.796732902526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1345 , TFLOPS: 10.431484788561434, Tokens per sec: 147355.73777454038, Loss: 8.787531852722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1346 , TFLOPS: 10.21372410952986, Tokens per sec: 144279.63823862688, Loss: 8.794718742370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1347 , TFLOPS: 10.334568203112026, Tokens per sec: 145986.68866590853, Loss: 8.792426109313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1348 , TFLOPS: 10.417094503042437, Tokens per sec: 147152.4597961491, Loss: 8.793598175048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1349 , TFLOPS: 10.363382680320095, Tokens per sec: 146393.7235831466, Loss: 8.79590129852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1350 , TFLOPS: 10.335804066219698, Tokens per sec: 146004.1465373152, Loss: 8.785027503967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1351 , TFLOPS: 9.469174256810795, Tokens per sec: 133762.08536085737, Loss: 8.798062324523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1352 , TFLOPS: 10.27296189062195, Tokens per sec: 145116.43444874295, Loss: 8.788613319396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1353 , TFLOPS: 10.279863604525058, Tokens per sec: 145213.9284454956, Loss: 8.790590286254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1354 , TFLOPS: 10.449647661381078, Tokens per sec: 147612.3075322204, Loss: 8.788108825683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1355 , TFLOPS: 10.197181584894407, Tokens per sec: 144045.9576100568, Loss: 8.797118186950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1356 , TFLOPS: 10.409133250935627, Tokens per sec: 147039.99870344877, Loss: 8.788152694702148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1357 , TFLOPS: 10.395389593042259, Tokens per sec: 146845.8550230766, Loss: 8.787029266357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1358 , TFLOPS: 10.404643050441374, Tokens per sec: 146976.56988003606, Loss: 8.786140441894531 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1359 , TFLOPS: 10.136669480535442, Tokens per sec: 143191.16023817324, Loss: 8.793267250061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1360 , TFLOPS: 10.476765137817145, Tokens per sec: 147995.3705216044, Loss: 8.794092178344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1361 , TFLOPS: 10.488586651341027, Tokens per sec: 148162.3618830675, Loss: 8.781444549560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1362 , TFLOPS: 10.51012016960312, Tokens per sec: 148466.54556685826, Loss: 8.78850269317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1363 , TFLOPS: 10.079765160830283, Tokens per sec: 142387.3266341683, Loss: 8.795367240905762 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1364 , TFLOPS: 10.16920158210572, Tokens per sec: 143650.71052515646, Loss: 8.787700653076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1365 , TFLOPS: 10.499871020673215, Tokens per sec: 148321.76551562498, Loss: 8.793656349182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1366 , TFLOPS: 9.706401081142745, Tokens per sec: 137113.16475443298, Loss: 8.784977912902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1367 , TFLOPS: 10.423497962722674, Tokens per sec: 147242.91542587167, Loss: 8.780324935913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1368 , TFLOPS: 10.387188150976918, Tokens per sec: 146730.00099358326, Loss: 8.786319732666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1369 , TFLOPS: 10.343962790076816, Tokens per sec: 146119.39712700908, Loss: 8.782262802124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1370 , TFLOPS: 10.463397303640814, Tokens per sec: 147806.53574808672, Loss: 8.790675163269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1371 , TFLOPS: 10.397949450182194, Tokens per sec: 146882.01570826172, Loss: 8.786681175231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1372 , TFLOPS: 10.506187018464777, Tokens per sec: 148410.98565381454, Loss: 8.783082962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1373 , TFLOPS: 10.486243762550082, Tokens per sec: 148129.26610491978, Loss: 8.788853645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1374 , TFLOPS: 10.45354929621933, Tokens per sec: 147667.42224424565, Loss: 8.784480094909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1375 , TFLOPS: 10.421664086858543, Tokens per sec: 147217.0099928079, Loss: 8.794910430908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1376 , TFLOPS: 10.43763408393114, Tokens per sec: 147442.60306499168, Loss: 8.780077934265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1377 , TFLOPS: 10.41373649124627, Tokens per sec: 147105.02433363255, Loss: 8.78969669342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1378 , TFLOPS: 10.413812004948959, Tokens per sec: 147106.09104442177, Loss: 8.791142463684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1379 , TFLOPS: 10.434202635243636, Tokens per sec: 147394.1302288392, Loss: 8.795583724975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1380 , TFLOPS: 10.435531442017682, Tokens per sec: 147412.90102767738, Loss: 8.78508186340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1381 , TFLOPS: 9.552203886933523, Tokens per sec: 134934.96656155642, Loss: 8.783854484558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1382 , TFLOPS: 10.342575920892335, Tokens per sec: 146099.80613531245, Loss: 8.783615112304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1383 , TFLOPS: 10.433742285290075, Tokens per sec: 147387.6272996382, Loss: 8.783267974853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1384 , TFLOPS: 10.542052998426183, Tokens per sec: 148917.6304934845, Loss: 8.788971900939941 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1385 , TFLOPS: 10.413434270089832, Tokens per sec: 147100.75514066714, Loss: 8.776199340820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1386 , TFLOPS: 10.454477072290562, Tokens per sec: 147680.52806093777, Loss: 8.785284042358398 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1387 , TFLOPS: 10.462320278341661, Tokens per sec: 147791.32162845033, Loss: 8.791889190673828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1388 , TFLOPS: 10.381029066700261, Tokens per sec: 146642.99742448467, Loss: 8.781611442565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1389 , TFLOPS: 10.4535494936912, Tokens per sec: 147667.4250337443, Loss: 8.785419464111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1390 , TFLOPS: 10.475126220053667, Tokens per sec: 147972.21907757854, Loss: 8.791377067565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1391 , TFLOPS: 10.347302388682904, Tokens per sec: 146166.5724837723, Loss: 8.783975601196289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1392 , TFLOPS: 10.262389456263831, Tokens per sec: 144967.08764946257, Loss: 8.79289722442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1393 , TFLOPS: 10.443185633707467, Tokens per sec: 147521.02456774644, Loss: 8.781366348266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1394 , TFLOPS: 10.379566798826376, Tokens per sec: 146622.34134668292, Loss: 8.77656364440918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1395 , TFLOPS: 9.656351489608177, Tokens per sec: 136406.1614240943, Loss: 8.787120819091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1396 , TFLOPS: 10.38811286688666, Tokens per sec: 146743.06358226296, Loss: 8.7765531539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1397 , TFLOPS: 10.404739439205132, Tokens per sec: 146977.93147310597, Loss: 8.785643577575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1398 , TFLOPS: 10.412830626811877, Tokens per sec: 147092.22804194834, Loss: 8.787338256835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1399 , TFLOPS: 10.415234306473323, Tokens per sec: 147126.18255533406, Loss: 8.789093017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1400 , TFLOPS: 10.45136252319994, Tokens per sec: 147636.53176622174, Loss: 8.779302597045898 +------------------------------------------------------------------ + eval ppl=6528.5322265625, eval loss=8.783937454223633 +Saving checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400 +Saving model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/model.pt +Saved model state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/rng.pt +Saved RNG states to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400/rng.pt +Saved checkpoint to /work/llm_recipes/models/tiny-mistral-sample/iter_0001400, took 0.18s +[rank0]:[2024-08-02 18:01:10,651] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0010275570002704626, 'preprocessing_with_comm': 0.0004898459997093596, 'state_converting': 0.02634215000034601, : 0.02818033599987757}) +------------------------------------------------------------------ +iteration: 1401 , TFLOPS: 10.358956186890554, Tokens per sec: 146331.1946892945, Loss: 8.781364440917969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1402 , TFLOPS: 10.444158587435911, Tokens per sec: 147534.76856655214, Loss: 8.7919921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1403 , TFLOPS: 10.41894623787512, Tokens per sec: 147178.61750600187, Loss: 8.780084609985352 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1404 , TFLOPS: 10.458409760504756, Tokens per sec: 147736.08143468908, Loss: 8.782623291015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1405 , TFLOPS: 10.4208504092257, Tokens per sec: 147205.51593704033, Loss: 8.77851676940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1406 , TFLOPS: 10.4442425966689, Tokens per sec: 147535.95528567772, Loss: 8.780743598937988 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1407 , TFLOPS: 10.396236231450326, Tokens per sec: 146857.81468459917, Loss: 8.783707618713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1408 , TFLOPS: 10.462291041520515, Tokens per sec: 147790.90862747902, Loss: 8.777971267700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1409 , TFLOPS: 10.433319462387749, Tokens per sec: 147381.6544796626, Loss: 8.781469345092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1410 , TFLOPS: 9.093190310165799, Tokens per sec: 128450.91509389698, Loss: 8.780712127685547 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240802_173428-s75vpwte/files/requirements.txt b/wandb/run-20240802_173428-s75vpwte/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240802_173428-s75vpwte/files/wandb-metadata.json b/wandb/run-20240802_173428-s75vpwte/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ba111d7b60468965d982586540753444149be89d --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-02T08:34:28.941229", + "startedAt": "2024-08-02T08:34:28.326109", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "512", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/custom/tiny-mistral", + "--save", + "/work/llm_recipes/models/tiny-mistral-sample", + "--load", + "/work/llm_recipes/models/tiny-mistral-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-mistral-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-mistral-sample_train_2024-08-02-17:34:15" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0409999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.041, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json b/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6f9fbe1bc9b1cfab6a8720882866d38a740805ab --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 8.780712127685547, "training/perplexity": 6507.50970149773, "utils/batch_size": 8, "utils/global_batch_size": 320, "utils/seq_len": 513, "utils/gradient_accumulation_steps": 40, "utils/iteration": 1410, "optimizer/lr": 1.989808738231659e-05, "optimizer/variance_l2": 0.013855160145659429, "optimizer/variance_sqrt_l2": 0.9992841304001847, "optimizer/momentum_l2": 0.9839698623853019, "optimizer/weight_l2": 101.83051175850979, "optimizer/variance_l1": 1.002197265625, "optimizer/variance_sqrt_l1": 536.5, "optimizer/momentum_l1": 403.875, "optimizer/weight_l1": 332288.0, "optimizer/variance_abs_max": 0.0011444091796875, "optimizer/variance_sqrt_abs_max": 0.033935546875, "optimizer/momentum_abs_max": 0.03369140625, "optimizer/weight_abs_max": 1.0, "stats/1_iteration_time": 1.277997902000152, "stats/tokens_per_sec": 128450.91509389698, "stats/tokens_per_sec_per_gpu": 128450.91509389698, "stats/tflops": 9.093190310165799, "_timestamp": 1722589282.0763872, "_runtime": 1613.73472905159, "_step": 1410, "evaluation/val_loss": 8.783937454223633, "evaluation/val_ppl": 6528.5322265625, "_wandb": {"runtime": 1614}} \ No newline at end of file diff --git a/wandb/run-20240802_173428-s75vpwte/logs/debug-internal.log b/wandb/run-20240802_173428-s75vpwte/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..28685671544fff62b132c8993d36f3836c0a096b --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/logs/debug-internal.log @@ -0,0 +1,8660 @@ +2024-08-02 17:34:28,342 INFO StreamThr :14040 [internal.py:wandb_internal():86] W&B internal server running at pid: 14040, started at: 2024-08-02 17:34:28.341198 +2024-08-02 17:34:28,343 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status +2024-08-02 17:34:28,346 INFO WriterThread:14040 [datastore.py:open_for_write():87] open: /project/wandb/run-20240802_173428-s75vpwte/run-s75vpwte.wandb +2024-08-02 17:34:28,347 DEBUG SenderThread:14040 [sender.py:send():382] send: header +2024-08-02 17:34:28,364 DEBUG SenderThread:14040 [sender.py:send():382] send: run +2024-08-02 17:34:28,827 INFO SenderThread:14040 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240802_173428-s75vpwte/files +2024-08-02 17:34:28,827 INFO SenderThread:14040 [sender.py:_start_run_threads():1136] run started: s75vpwte with start time 1722587668.341658 +2024-08-02 17:34:28,832 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: check_version +2024-08-02 17:34:28,832 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: check_version +2024-08-02 17:34:28,922 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: run_start +2024-08-02 17:34:28,928 DEBUG HandlerThread:14040 [system_info.py:__init__():27] System info init +2024-08-02 17:34:28,928 DEBUG HandlerThread:14040 [system_info.py:__init__():42] System info init done +2024-08-02 17:34:28,928 INFO HandlerThread:14040 [system_monitor.py:start():194] Starting system monitor +2024-08-02 17:34:28,928 INFO SystemMonitor:14040 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-02 17:34:28,929 INFO HandlerThread:14040 [system_monitor.py:probe():214] Collecting system info +2024-08-02 17:34:28,929 INFO SystemMonitor:14040 [interfaces.py:start():190] Started cpu monitoring +2024-08-02 17:34:28,930 INFO SystemMonitor:14040 [interfaces.py:start():190] Started disk monitoring +2024-08-02 17:34:28,930 INFO SystemMonitor:14040 [interfaces.py:start():190] Started gpu monitoring +2024-08-02 17:34:28,931 INFO SystemMonitor:14040 [interfaces.py:start():190] Started memory monitoring +2024-08-02 17:34:28,932 INFO SystemMonitor:14040 [interfaces.py:start():190] Started network monitoring +2024-08-02 17:34:28,941 DEBUG HandlerThread:14040 [system_info.py:probe():151] Probing system +2024-08-02 17:34:28,943 DEBUG HandlerThread:14040 [system_info.py:_probe_git():136] Probing git +2024-08-02 17:34:28,954 DEBUG HandlerThread:14040 [system_info.py:_probe_git():144] Probing git done +2024-08-02 17:34:28,954 DEBUG HandlerThread:14040 [system_info.py:probe():199] Probing system done +2024-08-02 17:34:28,954 DEBUG HandlerThread:14040 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-02T08:34:28.941229', 'startedAt': '2024-08-02T08:34:28.326109', 'docker': None, 'cuda': None, 'args': ('--seq-length', '512', '--sliding-window-size', '4096', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/custom/tiny-mistral', '--save', '/work/llm_recipes/models/tiny-mistral-sample', '--load', '/work/llm_recipes/models/tiny-mistral-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-mistral-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-mistral-sample_train_2024-08-02-17:34:15'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0409999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}, {'current': 2400.041, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-02 17:34:28,954 INFO HandlerThread:14040 [system_monitor.py:probe():224] Finished collecting system info +2024-08-02 17:34:28,954 INFO HandlerThread:14040 [system_monitor.py:probe():227] Publishing system info +2024-08-02 17:34:28,956 INFO HandlerThread:14040 [system_monitor.py:probe():229] Finished publishing system info +2024-08-02 17:34:28,967 DEBUG SenderThread:14040 [sender.py:send():382] send: files +2024-08-02 17:34:28,967 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-02 17:34:28,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: python_packages +2024-08-02 17:34:28,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:34:28,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:34:28,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: python_packages +2024-08-02 17:34:28,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:34:29,212 DEBUG SenderThread:14040 [sender.py:send():382] send: telemetry +2024-08-02 17:34:29,732 INFO wandb-upload_0:14040 [upload_job.py:push():131] Uploaded file /tmp/tmpmpfjyw2lwandb/83hwnqg1-wandb-metadata.json +2024-08-02 17:34:29,829 INFO Thread-12 :14040 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:29,829 INFO Thread-12 :14040 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240802_173428-s75vpwte/files/requirements.txt +2024-08-02 17:34:29,830 INFO Thread-12 :14040 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-metadata.json +2024-08-02 17:34:31,830 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:32,830 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:33,328 DEBUG SenderThread:14040 [sender.py:send():382] send: config +2024-08-02 17:34:33,328 DEBUG SenderThread:14040 [sender.py:send():382] send: config +2024-08-02 17:34:33,831 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:34,328 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:34:34,831 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:35,832 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:36,805 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:37,833 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:37,951 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:37,954 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:37,955 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:37,956 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:38,834 INFO Thread-12 :14040 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:39,091 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:39,094 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:39,094 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:39,095 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:39,835 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:39,835 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:40,096 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:34:40,230 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:40,233 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:40,233 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:40,234 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:40,836 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:41,375 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:41,377 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:41,378 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:41,379 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:41,837 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:41,837 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:42,510 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:42,513 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:42,513 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:42,514 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:42,838 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:43,641 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:43,643 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:43,644 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:43,645 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:43,839 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:43,839 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:43,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:34:43,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:34:43,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:34:44,771 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:44,772 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:44,773 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:44,774 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:44,840 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:45,819 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:34:45,840 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:45,907 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:45,909 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:45,910 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:45,911 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:46,841 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:47,046 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:47,048 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:47,049 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:47,050 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:47,842 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:47,842 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:48,184 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:48,186 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:48,187 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:48,188 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:48,843 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:49,330 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:49,334 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:49,334 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:49,335 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:49,845 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:49,845 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:50,467 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:50,468 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:50,469 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:50,470 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:50,846 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:51,470 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:34:51,602 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:51,605 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:51,605 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:51,606 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:51,847 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:51,847 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:52,740 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:52,742 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:52,743 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:52,744 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:52,848 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:53,849 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:53,879 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:53,881 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:53,881 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:53,882 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:54,850 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:55,106 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:55,108 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:55,109 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:55,110 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:55,852 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:55,852 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:56,244 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:56,247 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:56,247 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:56,264 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:56,853 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:57,265 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:34:57,381 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:57,383 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:57,384 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:57,385 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:57,854 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:57,854 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:58,518 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:58,520 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:58,521 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:58,522 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:58,856 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:34:58,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:34:58,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:34:58,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:34:59,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:34:59,653 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:34:59,653 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:34:59,654 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:34:59,857 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:34:59,857 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:00,787 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:00,789 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:00,790 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:00,791 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:00,858 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:01,859 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:01,932 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:01,934 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:01,935 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:01,936 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:02,860 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:02,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:03,064 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:03,170 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:03,170 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:03,171 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:03,861 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/config.yaml +2024-08-02 17:35:03,861 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:03,861 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:04,203 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:04,205 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:04,206 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:04,207 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:04,862 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:05,344 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:05,347 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:05,348 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:05,351 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:05,863 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:05,863 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:06,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:06,486 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:06,487 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:06,488 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:06,864 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:07,636 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:07,638 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:07,638 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:07,639 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:07,865 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:07,865 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:08,640 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:08,785 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:08,787 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:08,787 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:08,788 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:08,866 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:09,867 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:09,946 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:09,948 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:09,948 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:09,950 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:10,868 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:11,094 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:11,096 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:11,096 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:11,097 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:11,869 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:11,869 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:12,321 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:12,323 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:12,324 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:12,325 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:12,870 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:13,462 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:13,526 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:13,527 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:13,528 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:13,871 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:13,871 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:13,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:35:13,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:35:13,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:35:14,177 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:14,608 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:14,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:14,610 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:14,611 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:14,872 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:15,754 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:15,756 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:15,756 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:15,758 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:15,873 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:15,873 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:16,901 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:16,903 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:16,904 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:16,905 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:17,875 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:17,875 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:18,047 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:18,049 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:18,050 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:18,051 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:18,876 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:19,191 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:19,193 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:19,194 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:19,194 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:19,195 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:19,877 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:19,877 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:20,382 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:20,384 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:20,385 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:20,386 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:20,878 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:21,525 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:21,527 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:21,528 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:21,529 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:21,879 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:21,879 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:22,672 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:22,674 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:22,674 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:22,675 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:22,880 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:23,823 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:23,825 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:23,825 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:23,826 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:23,881 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:23,881 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:24,827 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:24,962 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:24,964 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:24,964 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:24,966 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:25,882 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:25,882 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:26,104 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:26,107 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:26,107 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:26,108 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:26,883 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:27,242 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:27,244 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:27,244 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:27,245 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:27,884 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:27,884 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:28,478 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:28,481 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:28,481 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:28,482 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:28,885 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:28,933 DEBUG SystemMonitor:14040 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-02 17:35:28,934 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:35:28,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:35:28,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:35:28,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:35:29,631 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:29,632 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:29,633 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:29,634 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:29,886 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:29,887 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:30,675 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:30,785 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:30,787 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:30,787 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:30,788 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:30,887 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:31,888 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:31,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:31,941 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:31,942 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:31,943 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:32,889 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:33,145 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:33,147 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:33,147 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:33,149 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:33,890 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:33,890 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:34,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:34,295 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:34,295 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:34,296 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:34,891 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:35,438 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:35,440 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:35,441 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:35,442 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:35,892 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:35,892 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:36,443 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:36,585 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:36,587 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:36,588 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:36,589 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:36,893 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:37,734 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:37,736 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:37,736 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:37,738 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:37,894 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:37,894 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:38,883 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:38,887 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:38,888 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:38,889 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:38,895 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:39,895 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:40,032 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:40,034 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:40,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:40,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:40,896 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:41,181 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:41,183 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:41,183 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:41,185 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:41,897 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:41,897 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:42,185 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:42,363 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:42,365 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:42,366 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:42,367 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:42,898 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:43,518 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:43,521 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:43,521 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:43,522 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:43,899 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:43,899 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:43,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:35:43,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:35:43,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:35:44,671 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:44,672 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:44,673 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:44,674 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:44,900 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:45,901 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:45,911 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:45,913 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:45,913 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:45,915 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:46,902 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:47,062 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:47,064 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:47,064 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:47,065 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:47,903 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:47,903 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:48,066 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:48,211 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:48,213 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:48,213 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:48,214 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:48,904 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:49,355 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:49,357 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:49,358 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:49,359 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:49,905 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:49,905 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:50,509 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:50,510 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:50,511 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:50,512 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:50,906 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:51,656 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:51,657 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:51,658 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:51,659 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:51,907 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:51,907 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:52,806 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:52,809 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:52,809 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:52,810 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:52,907 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:53,811 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:53,908 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:53,953 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:53,955 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:53,955 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:53,956 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:54,909 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:55,099 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:55,101 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:55,101 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:55,102 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:55,910 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:55,910 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:56,243 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:56,246 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:56,246 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:56,247 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:56,911 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:57,392 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:57,395 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:57,395 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:57,396 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:57,912 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:57,912 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:58,545 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:58,547 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:58,548 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:58,549 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:58,913 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:35:58,935 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:35:58,935 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:35:58,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:35:58,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:35:58,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:35:59,696 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:35:59,697 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:35:59,697 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:35:59,698 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:35:59,914 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:35:59,914 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:00,848 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:00,850 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:00,851 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:00,852 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:00,915 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:01,915 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:01,995 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:01,997 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:01,998 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:01,999 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:02,917 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:03,236 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:03,239 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:03,239 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:03,240 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:03,918 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:03,918 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:04,241 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:04,389 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:04,391 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:04,391 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:04,392 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:04,919 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:05,539 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:05,541 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:05,541 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:05,543 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:05,920 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:05,920 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:06,683 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:06,686 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:06,686 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:06,687 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:06,921 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:07,824 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:07,827 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:07,827 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:07,828 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:07,922 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:07,922 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:08,969 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:08,971 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:08,972 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:08,973 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:09,924 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:09,924 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:09,973 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:10,113 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:10,115 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:10,115 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:10,116 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:10,925 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:11,264 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:11,267 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:11,267 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:11,268 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:11,926 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:11,926 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:12,417 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:12,419 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:12,419 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:12,420 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:12,927 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:13,561 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:13,563 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:13,563 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:13,564 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:13,928 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:13,928 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:13,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:36:13,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:36:13,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:36:14,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:14,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:14,706 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:14,707 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:14,929 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:15,747 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:15,851 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:15,853 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:15,854 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:15,855 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:15,930 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:15,930 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:16,994 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:16,996 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:16,996 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:16,997 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:17,931 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:17,932 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:18,142 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:18,144 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:18,145 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:18,146 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:18,933 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:19,385 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:19,388 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:19,388 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:19,389 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:19,934 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:19,934 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:20,529 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:20,531 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:20,532 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:20,533 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:20,935 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:21,533 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:21,672 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:21,674 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:21,675 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:21,676 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:21,936 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:21,936 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:22,807 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:22,810 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:22,810 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:22,811 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:22,937 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:23,937 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:23,952 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:23,954 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:23,954 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:23,956 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:24,939 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:25,108 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:25,110 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:25,110 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:25,111 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:25,940 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:25,940 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:26,257 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:26,258 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:26,259 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:26,261 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:26,941 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:27,261 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:27,402 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:27,404 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:27,405 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:27,406 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:27,942 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:27,942 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:28,547 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:28,549 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:28,550 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:28,551 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:28,936 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:36:28,943 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:28,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:36:28,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:36:28,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:36:29,696 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:29,698 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:29,698 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:29,699 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:29,944 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:29,944 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:30,845 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:30,847 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:30,847 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:30,848 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:30,945 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:31,945 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:31,990 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:31,992 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:31,992 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:31,993 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:32,946 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:32,994 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:33,145 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:33,147 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:33,147 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:33,148 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:33,947 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:33,947 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:34,298 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:34,300 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:34,301 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:34,302 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:34,948 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:35,455 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:35,457 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:35,457 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:35,458 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:35,950 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:35,950 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:36,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:36,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:36,706 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:36,708 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:36,951 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:37,867 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:37,869 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:37,869 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:37,871 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:37,952 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:37,952 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:38,871 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:39,023 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:39,025 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:39,025 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:39,026 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:39,953 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:39,953 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:40,177 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:40,179 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:40,179 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:40,180 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:40,954 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:41,335 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:41,337 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:41,338 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:41,339 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:41,955 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:41,955 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:42,487 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:42,490 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:42,490 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:42,491 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:42,956 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:43,643 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:43,645 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:43,646 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:43,647 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:43,957 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:43,957 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:43,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:36:43,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:36:43,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:36:44,223 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:44,797 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:44,799 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:44,799 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:44,800 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:44,958 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:45,943 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:45,945 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:45,946 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:45,947 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:45,959 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:45,960 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:47,083 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:47,085 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:47,085 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:47,086 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:47,961 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:47,961 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:48,216 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:48,219 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:48,219 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:48,220 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:48,962 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:49,349 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:49,351 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:49,352 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:49,352 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:49,353 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:49,963 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:49,963 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:50,481 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:50,483 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:50,484 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:50,485 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:50,964 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:51,614 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:51,616 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:51,617 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:51,618 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:51,965 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:51,965 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:52,839 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:52,841 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:52,841 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:52,843 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:52,966 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:53,966 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:53,973 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:53,976 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:53,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:53,977 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:54,968 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:54,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:36:55,159 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:55,162 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:55,162 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:55,164 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:55,969 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:55,969 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:56,292 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:56,294 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:56,295 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:56,296 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:56,970 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:57,462 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:57,464 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:57,464 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:57,465 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:57,971 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:57,971 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:58,599 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:58,601 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:58,602 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:58,603 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:58,936 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:36:58,972 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:36:58,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:36:58,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:36:58,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:36:59,737 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:36:59,738 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:36:59,739 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:36:59,740 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:36:59,973 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:36:59,973 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:00,779 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:00,871 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:00,873 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:00,873 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:00,874 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:00,974 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:01,974 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:02,010 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:02,012 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:02,012 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:02,013 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:02,975 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:03,148 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:03,151 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:03,151 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:03,152 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:03,977 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:03,977 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:04,281 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:04,283 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:04,284 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:04,285 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:04,978 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:05,415 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:05,417 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:05,417 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:05,418 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:05,979 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:05,979 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:06,419 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:06,541 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:06,543 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:06,544 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:06,545 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:06,980 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:07,665 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:07,668 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:07,668 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:07,669 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:07,981 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:07,982 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:08,795 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:08,797 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:08,798 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:08,799 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:08,983 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:09,983 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:10,014 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:10,017 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:10,017 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:10,018 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:10,985 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:11,148 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:11,150 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:11,150 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:11,151 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:11,986 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:11,986 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:12,152 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:12,272 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:12,275 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:12,275 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:12,276 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:12,987 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:13,403 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:13,406 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:13,406 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:13,407 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:13,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:37:13,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:37:13,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:37:13,989 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:13,989 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:14,537 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:14,562 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:14,562 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:14,563 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:14,990 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:15,670 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:15,672 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:15,673 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:15,673 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:15,991 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:15,991 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:16,803 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:16,805 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:16,806 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:16,807 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:16,993 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:17,807 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:17,936 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:17,939 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:17,939 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:17,940 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:17,994 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:17,994 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:19,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:19,074 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:19,075 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:19,076 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:19,996 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:19,996 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:20,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:20,209 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:20,209 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:20,210 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:20,997 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:21,345 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:21,348 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:21,348 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:21,349 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:21,998 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:21,998 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:22,477 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:22,480 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:22,480 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:22,481 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:23,000 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:23,482 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:23,611 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:23,613 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:23,614 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:23,615 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:24,001 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:24,001 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:24,742 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:24,744 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:24,745 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:24,746 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:25,002 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:25,870 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:25,872 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:25,873 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:25,874 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:26,003 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:26,004 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:27,089 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:27,091 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:27,092 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:27,093 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:28,005 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:28,006 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:28,211 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:28,213 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:28,214 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:28,215 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:28,937 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:37:28,938 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:28,976 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:37:28,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:37:28,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:37:29,007 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:29,334 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:29,335 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:29,335 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:29,336 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:30,011 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:30,011 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:30,454 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:30,456 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:30,457 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:30,458 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:31,012 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:31,576 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:31,578 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:31,578 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:31,579 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:32,013 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:32,014 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:32,697 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:32,699 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:32,699 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:32,700 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:33,015 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:33,818 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:33,820 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:33,820 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:33,821 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:34,016 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:34,016 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:34,822 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:34,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:34,941 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:34,942 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:34,943 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:35,017 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:36,017 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:36,061 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:36,063 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:36,063 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:36,064 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:37,018 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:37,190 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:37,192 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:37,193 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:37,194 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:38,019 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:38,020 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:38,325 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:38,328 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:38,328 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:38,329 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:39,021 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:39,457 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:39,459 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:39,460 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:39,461 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:40,022 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:40,022 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:40,461 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:40,588 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:40,591 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:40,591 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:40,592 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:41,023 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:41,718 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:41,720 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:41,721 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:41,722 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:42,024 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:42,024 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:42,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:42,942 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:42,942 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:42,943 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:43,025 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:43,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:37:43,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:37:43,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:37:44,025 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:44,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:44,152 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:44,152 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:44,153 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:45,026 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:45,204 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:45,206 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:45,206 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:45,207 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:46,028 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:46,028 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:46,247 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:46,384 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:46,387 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:46,387 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:46,388 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:47,029 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:47,515 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:47,517 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:47,518 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:47,519 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:48,030 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:48,030 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:48,646 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:48,648 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:48,649 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:48,650 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:49,031 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:49,776 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:49,778 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:49,779 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:49,780 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:50,031 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:50,032 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:50,906 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:50,908 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:50,909 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:50,910 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:51,032 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:51,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:52,033 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:52,035 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:52,037 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:52,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:52,039 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:53,034 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:53,164 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:53,166 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:53,166 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:53,167 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:54,035 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:54,035 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:54,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:54,295 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:54,295 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:54,296 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:55,036 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:55,422 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:55,425 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:55,425 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:55,426 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:56,037 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:56,037 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:56,565 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:56,568 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:56,568 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:56,569 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:57,038 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:57,570 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:37:57,706 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:57,708 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:57,708 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:57,709 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:58,039 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:37:58,039 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:37:58,839 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:37:58,841 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:37:58,841 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:37:58,842 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:37:58,938 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:37:58,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:37:58,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:37:58,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:37:59,040 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:00,040 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:00,064 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:00,065 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:00,066 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:00,067 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:01,041 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:01,199 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:01,201 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:01,201 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:01,203 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:02,042 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:02,043 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:02,340 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:02,343 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:02,343 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:02,344 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:03,044 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:03,345 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:03,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:03,485 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:03,486 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:03,487 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:04,045 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:04,045 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:04,631 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:04,633 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:04,633 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:04,635 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:05,046 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:05,771 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:05,773 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:05,774 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:05,775 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:06,046 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:06,047 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:06,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:06,912 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:06,912 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:06,913 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:07,047 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:08,048 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:08,050 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:08,052 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:08,053 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:08,054 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:09,049 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:09,054 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:09,189 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:09,191 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:09,191 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:09,192 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:10,050 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:10,050 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:10,328 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:10,330 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:10,331 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:10,332 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:11,051 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:11,469 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:11,471 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:11,472 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:11,473 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:12,052 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:12,052 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:12,610 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:12,613 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:12,613 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:12,614 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:13,053 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:13,751 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:13,753 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:13,754 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:13,755 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:13,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:38:13,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:38:13,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:38:14,054 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:14,054 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:14,150 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:14,895 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:14,897 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:14,897 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:14,898 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:15,054 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:16,055 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:16,139 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:16,142 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:16,142 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:16,143 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:17,056 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:17,295 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:17,297 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:17,297 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:17,298 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:18,057 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:18,057 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:18,449 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:18,451 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:18,452 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:18,453 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:19,058 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:19,453 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:19,593 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:19,595 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:19,595 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:19,597 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:20,059 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:20,060 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:20,732 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:20,734 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:20,734 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:20,736 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:21,060 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:21,872 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:21,873 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:21,874 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:21,875 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:22,061 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:22,062 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:23,012 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:23,014 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:23,014 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:23,016 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:23,062 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:24,063 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:24,153 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:24,155 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:24,155 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:24,157 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:25,064 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:25,157 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:25,292 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:25,294 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:25,295 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:25,296 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:25,480 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:26,065 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:26,065 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:26,800 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:26,803 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:26,803 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:26,804 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:27,066 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:27,066 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:27,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:27,941 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:27,942 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:27,943 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:28,067 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:28,067 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:28,939 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:38:28,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:38:28,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:38:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:38:29,085 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:29,185 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:29,185 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:29,187 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:30,069 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:30,069 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:30,187 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:30,217 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:30,219 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:30,219 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:30,220 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:31,070 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:31,351 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:31,354 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:31,354 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:31,355 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:32,071 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:32,071 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:32,481 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:32,484 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:32,484 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:32,485 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:33,072 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:33,748 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:33,751 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:33,751 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:33,752 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:34,073 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:34,073 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:34,892 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:34,896 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:34,896 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:34,897 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:35,074 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:35,897 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:36,023 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:36,025 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:36,025 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:36,026 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:36,075 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:36,075 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:37,155 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:37,157 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:37,157 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:37,158 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:38,076 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:38,077 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:38,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:38,295 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:38,296 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:38,297 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:39,077 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:39,474 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:39,476 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:39,476 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:39,477 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:40,078 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:40,079 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:40,608 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:40,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:40,611 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:40,612 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:41,079 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:41,612 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:41,742 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:41,745 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:41,745 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:41,746 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:42,080 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:42,081 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:42,878 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:42,880 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:42,880 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:42,881 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:43,081 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:43,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:38:43,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:38:43,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:38:44,012 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:44,082 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:44,184 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:44,185 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:44,186 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:45,083 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:45,156 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:45,158 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:45,158 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:45,159 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:46,084 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:46,084 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:46,284 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:46,286 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:46,287 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:46,288 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:47,085 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:47,288 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:47,406 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:47,408 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:47,408 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:47,409 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:48,086 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:48,086 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:48,537 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:48,539 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:48,540 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:48,541 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:49,087 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:49,675 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:49,677 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:49,678 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:49,679 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:50,088 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:50,088 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:50,903 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:50,905 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:50,906 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:50,907 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:51,089 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:52,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:52,041 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:52,041 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:52,042 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:52,090 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:52,090 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:53,042 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:53,172 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:53,174 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:53,174 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:53,175 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:54,092 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:54,092 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:54,305 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:54,307 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:54,307 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:54,308 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:55,093 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:55,500 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:55,502 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:55,503 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:55,504 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:56,094 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:56,094 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:56,639 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:56,642 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:56,642 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:56,643 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:57,095 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:57,769 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:57,771 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:57,772 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:57,773 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:58,096 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:38:58,096 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:38:58,773 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:38:58,907 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:38:58,909 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:38:58,910 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:38:58,911 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:38:58,940 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:38:58,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:38:58,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:38:58,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:38:59,097 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:00,053 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:00,054 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:00,055 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:00,056 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:00,098 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:00,098 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:01,197 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:01,199 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:01,199 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:01,201 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:02,099 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:02,100 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:02,342 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:02,344 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:02,344 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:02,345 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:03,100 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:03,469 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:03,472 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:03,472 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:03,473 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:04,101 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:04,102 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:04,473 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:04,609 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:04,612 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:04,612 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:04,613 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:05,102 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:05,793 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:05,795 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:05,795 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:05,796 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:06,103 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:06,104 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:06,933 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:06,935 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:06,936 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:06,937 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:07,104 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:08,105 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:08,215 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:08,217 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:08,218 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:08,219 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:09,106 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:09,363 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:09,365 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:09,365 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:09,367 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:10,107 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:10,107 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:10,367 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:10,506 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:10,508 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:10,509 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:10,510 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:11,108 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:11,658 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:11,660 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:11,660 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:11,661 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:12,109 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:12,109 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:12,807 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:12,809 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:12,810 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:12,811 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:13,110 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:13,956 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:39:13,982 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:13,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:39:13,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:13,984 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:13,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:39:14,111 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:14,111 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:15,094 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:15,096 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:15,096 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:15,097 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:15,112 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:16,113 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:16,135 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:16,235 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:16,237 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:16,237 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:16,239 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:17,114 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:17,371 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:17,373 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:17,373 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:17,374 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:18,114 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:18,115 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:18,504 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:18,506 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:18,507 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:18,508 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:19,115 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:19,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:19,653 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:19,653 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:19,654 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:20,116 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:20,117 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:20,788 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:20,790 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:20,791 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:20,792 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:21,117 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:21,792 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:21,927 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:21,929 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:21,929 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:21,931 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:22,118 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:22,118 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:23,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:23,074 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:23,074 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:23,076 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:23,119 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:24,120 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:24,215 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:24,217 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:24,218 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:24,219 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:25,121 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:25,452 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:25,455 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:25,455 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:25,456 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:26,122 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:26,122 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:26,601 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:26,603 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:26,604 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:26,605 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:27,123 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:27,606 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:27,750 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:27,752 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:27,752 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:27,753 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:28,124 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:28,124 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:28,894 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:28,896 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:28,897 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:28,898 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:28,940 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:39:28,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:39:28,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:39:28,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:39:29,125 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:30,033 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:30,035 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:30,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:30,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:30,126 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:30,126 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:31,181 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:31,184 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:31,184 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:31,185 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:32,128 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:32,128 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:32,328 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:32,331 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:32,331 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:32,332 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:33,129 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:33,332 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:33,482 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:33,484 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:33,485 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:33,486 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:34,130 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:34,130 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:34,630 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:34,633 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:34,633 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:34,634 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:35,131 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:35,779 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:35,781 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:35,781 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:35,782 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:36,132 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:36,132 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:36,940 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:36,943 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:36,943 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:36,944 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:37,133 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:38,091 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:38,093 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:38,094 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:38,095 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:38,133 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:38,134 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:39,095 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:39,241 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:39,244 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:39,244 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:39,245 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:40,135 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:40,135 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:40,392 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:40,394 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:40,394 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:40,396 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:41,136 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:41,549 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:41,551 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:41,551 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:41,553 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:42,137 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:42,137 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:42,785 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:42,787 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:42,787 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:42,789 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:43,138 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:43,922 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:43,925 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:43,925 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:43,926 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:43,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:39:43,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:39:43,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:39:44,139 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:44,139 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:44,187 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:45,060 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:45,061 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:45,062 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:45,063 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:45,140 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:46,140 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:46,197 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:46,199 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:46,199 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:46,200 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:47,141 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:47,336 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:47,338 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:47,339 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:47,340 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:48,142 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:48,143 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:48,480 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:48,482 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:48,482 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:48,483 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:49,143 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:49,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:49,628 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:49,630 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:49,630 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:49,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:50,144 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:50,145 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:50,772 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:50,774 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:50,774 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:50,775 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:51,145 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:51,925 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:51,927 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:51,927 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:51,928 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:52,146 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:52,147 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:53,081 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:53,083 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:53,084 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:53,085 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:53,147 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:54,148 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:54,224 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:54,226 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:54,227 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:54,228 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:55,149 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:55,228 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:39:55,380 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:55,381 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:55,382 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:55,383 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:56,150 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:56,150 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:56,530 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:56,532 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:56,533 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:56,534 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:57,151 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:57,677 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:57,680 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:57,680 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:57,681 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:58,152 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:39:58,152 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:39:58,920 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:39:58,923 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:39:58,923 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:39:58,924 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:39:58,941 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:39:58,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:39:58,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:39:58,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:39:59,153 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:00,056 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:00,057 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:00,058 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:00,059 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:00,154 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:00,154 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:01,099 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:01,238 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:01,240 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:01,240 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:01,241 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:02,156 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:02,156 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:02,374 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:02,376 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:02,376 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:02,377 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:03,157 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:03,506 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:03,509 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:03,509 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:03,510 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:04,158 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:04,158 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:04,640 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:04,642 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:04,642 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:04,644 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:05,159 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:05,772 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:05,775 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:05,775 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:05,776 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:06,160 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:06,160 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:06,777 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:06,906 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:06,908 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:06,908 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:06,909 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:07,161 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:08,036 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:08,039 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:08,039 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:08,040 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:08,162 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:08,162 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:09,165 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:09,168 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:09,168 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:09,169 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:10,163 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:10,163 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:10,299 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:10,301 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:10,302 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:10,303 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:11,164 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:11,429 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:11,431 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:11,432 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:11,433 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:12,165 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:12,166 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:12,434 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:12,557 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:12,559 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:12,560 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:12,561 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:13,166 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:13,692 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:13,694 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:13,694 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:13,695 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:13,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:40:13,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:40:13,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:40:14,167 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:14,167 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:14,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:14,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:14,828 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:14,829 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:15,168 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:16,062 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:16,064 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:16,065 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:16,066 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:16,169 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:16,170 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:17,194 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:17,196 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:17,197 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:17,198 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:18,171 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:18,171 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:18,199 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:18,334 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:18,337 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:18,337 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:18,338 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:19,172 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:19,468 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:19,470 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:19,471 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:19,472 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:20,173 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:20,173 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:20,600 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:20,602 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:20,603 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:20,604 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:21,174 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:21,731 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:21,733 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:21,734 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:21,735 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:22,175 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:22,175 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:22,854 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:22,856 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:22,856 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:22,857 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:23,176 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:23,858 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:23,975 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:23,977 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:23,977 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:23,978 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:24,177 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:24,177 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:25,098 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:25,101 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:25,101 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:25,102 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:25,178 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:26,178 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:26,272 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:26,274 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:26,275 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:26,276 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:27,179 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:27,396 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:27,399 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:27,399 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:27,400 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:28,180 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:28,180 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:28,528 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:28,530 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:28,531 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:28,532 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:28,942 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:40:28,943 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:28,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:40:28,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:40:28,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:40:29,181 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:29,674 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:29,676 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:29,676 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:29,677 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:30,182 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:30,182 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:30,818 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:30,821 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:30,821 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:30,822 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:31,183 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:32,044 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:32,047 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:32,047 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:32,049 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:32,184 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:32,185 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:33,172 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:33,174 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:33,175 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:33,176 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:33,185 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:34,177 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:34,186 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:34,309 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:34,311 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:34,312 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:34,313 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:35,187 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:35,452 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:35,454 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:35,455 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:35,456 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:36,188 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:36,188 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:36,591 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:36,593 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:36,593 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:36,595 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:37,189 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:37,721 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:37,723 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:37,724 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:37,725 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:38,190 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:38,190 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:38,853 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:38,857 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:38,857 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:38,858 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:39,191 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:39,858 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:39,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:39,984 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:39,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:39,986 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:40,192 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:40,192 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:41,124 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:41,126 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:41,126 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:41,128 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:41,193 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:42,194 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:42,262 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:42,264 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:42,264 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:42,265 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:43,195 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:43,409 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:43,411 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:43,412 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:43,413 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:43,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:40:43,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:40:43,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:40:44,196 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:44,196 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:44,560 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:44,562 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:44,562 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:44,563 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:45,197 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:45,603 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:45,705 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:45,707 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:45,707 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:45,709 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:46,199 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:46,199 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:46,850 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:46,852 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:46,852 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:46,853 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:47,200 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:47,989 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:47,991 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:47,992 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:47,993 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:48,201 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:48,201 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:49,222 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:49,225 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:49,225 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:49,226 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:50,203 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:50,203 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:50,365 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:50,367 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:50,368 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:50,369 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:51,204 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:51,370 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:51,505 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:51,508 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:51,508 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:51,509 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:52,205 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:52,205 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:52,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:52,653 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:52,653 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:52,654 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:53,206 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:53,797 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:53,799 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:53,800 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:53,801 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:54,208 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:54,208 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:54,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:54,943 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:54,943 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:54,944 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:55,209 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:56,086 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:56,088 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:56,088 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:56,089 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:56,210 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:56,210 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:57,090 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:40:57,236 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:57,238 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:57,239 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:57,240 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:58,212 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:40:58,212 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:58,372 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:58,374 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:58,374 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:58,375 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:40:58,944 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:40:58,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:40:58,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:40:58,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:40:59,213 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:40:59,561 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:40:59,563 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:40:59,563 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:40:59,564 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:00,214 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:00,214 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:00,694 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:00,696 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:00,697 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:00,698 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:01,215 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:01,822 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:01,824 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:01,825 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:01,826 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:02,216 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:02,216 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:02,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:02,957 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:02,959 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:02,960 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:02,961 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:03,217 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:04,102 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:04,104 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:04,105 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:04,106 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:04,218 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:04,218 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:05,376 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:05,379 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:05,379 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:05,380 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:06,219 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:06,219 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:06,523 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:06,525 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:06,526 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:06,527 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:07,220 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:07,655 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:07,657 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:07,657 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:07,658 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:08,221 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:08,221 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:08,659 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:08,796 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:08,799 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:08,799 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:08,800 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:09,222 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:09,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:09,943 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:09,944 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:09,945 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:10,223 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:10,223 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:11,087 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:11,089 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:11,090 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:11,091 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:11,224 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:12,225 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:12,234 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:12,236 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:12,237 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:12,238 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:13,226 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:13,388 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:13,390 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:13,390 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:13,391 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:13,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:41:13,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:41:13,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:41:14,227 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:14,227 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:14,246 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:14,545 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:14,546 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:14,546 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:14,548 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:15,228 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:15,700 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:15,703 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:15,703 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:15,704 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:16,229 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:16,229 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:16,852 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:16,877 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:16,878 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:16,879 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:17,230 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:18,000 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:18,002 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:18,003 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:18,004 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:18,231 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:18,231 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:19,146 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:19,149 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:19,149 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:19,150 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:19,232 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:20,150 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:20,232 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:20,301 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:20,304 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:20,305 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:20,306 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:21,233 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:21,443 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:21,446 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:21,446 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:21,447 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:22,234 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:22,234 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:22,679 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:22,682 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:22,682 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:22,683 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:23,235 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:23,822 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:23,824 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:23,824 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:23,825 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:24,236 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:24,236 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:24,964 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:24,966 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:24,967 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:24,968 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:25,237 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:25,968 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:26,120 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:26,122 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:26,123 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:26,124 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:26,238 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:26,238 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:27,278 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:27,281 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:27,281 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:27,282 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:28,240 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:28,240 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:28,420 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:28,422 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:28,423 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:28,424 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:28,944 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:41:28,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:41:28,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:41:28,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:41:29,241 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:29,562 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:29,564 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:29,564 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:29,565 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:30,242 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:30,242 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:30,692 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:30,695 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:30,695 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:30,696 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:31,243 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:31,696 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:31,838 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:31,841 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:31,841 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:31,842 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:32,244 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:32,244 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:32,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:32,983 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:32,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:32,985 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:33,245 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:34,129 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:34,131 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:34,131 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:34,132 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:34,246 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:34,246 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:35,270 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:35,272 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:35,273 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:35,274 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:36,247 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:36,248 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:36,412 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:36,414 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:36,415 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:36,416 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:37,248 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:37,416 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:37,564 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:37,566 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:37,567 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:37,568 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:38,249 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:38,250 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:38,708 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:38,710 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:38,711 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:38,712 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:39,251 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:39,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:39,944 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:39,944 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:39,945 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:40,252 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:40,252 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:41,093 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:41,096 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:41,097 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:41,098 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:41,253 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:42,240 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:42,242 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:42,242 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:42,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:42,254 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:42,254 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:43,244 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:43,379 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:43,381 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:43,381 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:43,382 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:43,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:41:43,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:41:43,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:41:44,255 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:44,255 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:44,524 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:44,526 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:44,526 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:44,527 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:45,256 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:45,679 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:45,682 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:45,682 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:45,683 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:46,257 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:46,257 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:46,829 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:46,832 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:46,832 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:46,833 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:47,258 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:47,973 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:47,975 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:47,975 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:47,976 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:48,259 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:48,259 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:48,977 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:49,117 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:49,119 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:49,120 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:49,121 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:49,260 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:50,260 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:50,261 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:50,264 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:50,264 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:50,265 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:51,261 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:51,406 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:51,408 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:51,408 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:51,409 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:52,262 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:52,263 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:52,538 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:52,540 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:52,540 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:52,541 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:53,263 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:53,669 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:53,671 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:53,671 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:53,672 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:54,264 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:54,265 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:54,673 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:41:54,798 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:54,800 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:54,800 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:54,801 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:55,265 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:55,927 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:55,929 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:55,929 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:55,930 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:56,267 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:56,267 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:57,146 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:57,148 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:57,149 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:57,150 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:57,268 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:58,266 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:58,269 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:58,269 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:58,270 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:41:58,270 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:41:58,270 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:41:58,945 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:41:58,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:41:58,978 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:41:58,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:41:59,444 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:41:59,448 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:41:59,449 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:41:59,450 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:00,272 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:00,272 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:00,487 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:00,575 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:00,578 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:00,578 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:00,579 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:01,273 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:01,708 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:01,711 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:01,712 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:01,713 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:02,274 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:02,274 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:02,854 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:02,856 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:02,857 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:02,858 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:03,275 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:03,992 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:03,995 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:03,995 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:03,996 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:04,276 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:04,276 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:05,135 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:05,137 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:05,137 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:05,138 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:05,277 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:06,139 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:06,277 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:06,281 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:06,283 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:06,284 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:06,284 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:07,278 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:07,424 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:07,426 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:07,426 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:07,427 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:08,279 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:08,279 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:08,566 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:08,569 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:08,569 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:08,570 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:09,280 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:09,709 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:09,711 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:09,712 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:09,713 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:10,281 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:10,281 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:10,850 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:10,852 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:10,852 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:10,853 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:11,282 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:11,854 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:11,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:11,989 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:11,990 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:11,991 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:12,283 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:12,283 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:13,126 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:13,128 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:13,128 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:13,129 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:13,284 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:13,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:42:13,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:42:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:42:14,285 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:14,397 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:14,399 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:14,399 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:14,400 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:15,286 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:15,534 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:15,536 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:15,537 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:15,538 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:15,642 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:16,287 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:16,287 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:16,948 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:16,950 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:16,950 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:16,951 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:16,951 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:17,288 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:18,080 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:18,082 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:18,082 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:18,083 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:18,289 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:18,289 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:19,212 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:19,214 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:19,215 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:19,216 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:19,290 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:20,290 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:20,339 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:20,341 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:20,341 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:20,342 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:21,291 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:21,471 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:21,473 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:21,474 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:21,475 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:22,292 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:22,292 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:22,476 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:22,602 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:22,604 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:22,605 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:22,606 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:23,293 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:23,734 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:23,736 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:23,736 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:23,737 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:24,294 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:24,294 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:24,866 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:24,868 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:24,869 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:24,870 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:25,295 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:26,004 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:26,007 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:26,007 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:26,008 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:26,296 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:26,296 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:27,134 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:27,136 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:27,136 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:27,137 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:27,297 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:28,138 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:28,267 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:28,269 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:28,270 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:28,271 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:28,298 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:28,298 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:28,945 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:42:28,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:42:28,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:42:28,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:42:29,406 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:29,408 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:29,408 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:29,409 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:30,300 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:30,300 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:30,542 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:30,544 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:30,544 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:30,545 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:31,301 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:31,768 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:31,770 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:31,771 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:31,772 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:32,302 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:32,303 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:32,895 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:32,897 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:32,897 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:32,898 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:33,304 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:33,899 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:34,027 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:34,030 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:34,030 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:34,031 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:34,305 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:34,305 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:35,159 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:35,162 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:35,162 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:35,163 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:35,307 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:36,295 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:36,298 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:36,298 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:36,299 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:36,308 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:36,308 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:37,424 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:37,427 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:37,427 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:37,428 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:38,310 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:38,310 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:38,557 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:38,559 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:38,560 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:38,561 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:39,311 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:39,561 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:39,687 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:39,689 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:39,690 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:39,691 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:40,312 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:40,313 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:40,814 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:40,817 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:40,817 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:40,818 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:41,314 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:41,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:41,944 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:41,944 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:41,945 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:42,315 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:42,315 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:43,074 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:43,076 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:43,076 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:43,077 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:43,316 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:43,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:42:43,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:42:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:42:44,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:44,209 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:44,209 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:44,210 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:44,317 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:44,317 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:45,251 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:45,352 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:45,355 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:45,355 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:45,356 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:46,319 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:46,319 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:46,503 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:46,505 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:46,506 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:46,507 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:47,320 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:47,657 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:47,659 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:47,659 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:47,660 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:48,321 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:48,321 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:48,883 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:48,886 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:48,886 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:48,887 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:49,322 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:50,036 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:50,038 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:50,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:50,039 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:50,323 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:50,323 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:51,040 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:51,189 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:51,191 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:51,191 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:51,192 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:51,324 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:52,324 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:52,332 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:52,334 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:52,335 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:52,336 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:53,325 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:53,449 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:53,451 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:53,451 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:53,453 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:54,326 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:54,327 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:54,566 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:54,568 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:54,568 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:54,569 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:55,327 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:55,688 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:55,690 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:55,690 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:55,691 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:56,328 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:56,329 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:56,692 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:42:56,817 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:56,819 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:56,820 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:56,821 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:57,329 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:57,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:57,943 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:57,944 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:57,945 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:58,330 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:42:58,331 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:42:58,946 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:42:58,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:42:58,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:42:58,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:42:59,081 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:42:59,227 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:42:59,227 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:42:59,228 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:42:59,331 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:00,242 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:00,244 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:00,245 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:00,246 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:00,332 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:00,333 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:01,394 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:01,397 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:01,397 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:01,398 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:02,334 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:02,334 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:02,399 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:02,526 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:02,529 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:02,529 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:02,530 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:03,335 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:03,664 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:03,667 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:03,667 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:03,668 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:04,336 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:04,336 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:04,795 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:04,797 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:04,798 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:04,799 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:05,337 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:06,021 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:06,024 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:06,024 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:06,025 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:06,338 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:06,339 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:07,160 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:07,162 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:07,163 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:07,164 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:07,339 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:08,164 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:08,287 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:08,289 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:08,290 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:08,291 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:08,340 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:08,340 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:09,420 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:09,422 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:09,422 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:09,423 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:10,342 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:10,342 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:10,551 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:10,553 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:10,554 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:10,555 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:11,343 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:11,680 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:11,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:11,707 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:11,708 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:12,344 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:12,344 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:12,810 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:12,812 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:12,813 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:12,814 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:13,345 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:13,814 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:13,969 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:13,972 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:13,972 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:13,973 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:13,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:43:13,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:43:13,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:43:14,346 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:14,346 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:15,104 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:15,106 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:15,106 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:15,107 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:15,347 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:16,239 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:16,241 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:16,241 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:16,242 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:16,348 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:16,348 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:17,376 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:17,378 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:17,379 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:17,380 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:18,349 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:18,350 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:18,510 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:18,512 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:18,513 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:18,514 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:19,350 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:19,514 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:19,642 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:19,644 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:19,645 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:19,646 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:20,352 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:20,352 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:20,778 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:20,780 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:20,780 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:20,781 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:21,353 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:21,908 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:21,910 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:21,910 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:21,911 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:22,354 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:22,354 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:23,033 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:23,035 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:23,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:23,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:23,355 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:24,246 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:24,248 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:24,249 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:24,250 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:24,356 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:24,356 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:25,250 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:25,375 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:25,378 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:25,378 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:25,379 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:26,358 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:26,358 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:26,507 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:26,510 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:26,510 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:26,511 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:27,359 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:27,633 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:27,635 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:27,636 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:27,637 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:28,360 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:28,360 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:28,757 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:28,759 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:28,760 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:28,761 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:28,947 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:43:28,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:43:28,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:43:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:43:29,361 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:29,884 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:29,885 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:29,886 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:29,887 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:30,362 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:30,362 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:30,927 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:31,007 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:31,009 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:31,010 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:31,011 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:31,363 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:32,131 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:32,133 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:32,134 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:32,135 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:32,364 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:32,364 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:33,254 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:33,257 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:33,257 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:33,258 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:33,365 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:34,365 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:34,383 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:34,385 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:34,385 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:34,387 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:35,366 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:35,530 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:35,532 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:35,533 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:35,534 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:36,367 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:36,368 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:36,534 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:36,658 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:36,660 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:36,661 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:36,662 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:37,369 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:37,794 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:37,797 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:37,797 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:37,798 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:38,370 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:38,370 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:38,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:38,941 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:38,941 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:38,942 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:39,371 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:40,082 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:40,084 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:40,084 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:40,085 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:40,372 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:40,372 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:41,363 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:41,365 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:41,365 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:41,367 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:41,373 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:42,368 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:42,373 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:42,509 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:42,512 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:42,513 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:42,514 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:43,374 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:43,643 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:43,645 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:43,645 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:43,646 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:43,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:43:43,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:43:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:43:44,375 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:44,375 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:44,775 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:44,777 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:44,777 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:44,778 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:45,376 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:45,924 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:45,927 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:45,927 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:45,928 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:46,377 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:46,377 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:47,068 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:47,070 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:47,071 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:47,072 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:47,378 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:48,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:48,213 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:48,215 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:48,215 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:48,216 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:48,379 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:48,379 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:49,352 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:49,354 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:49,354 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:49,356 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:49,380 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:50,381 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:50,500 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:50,502 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:50,503 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:50,504 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:51,382 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:51,642 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:51,645 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:51,645 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:51,646 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:52,383 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:52,383 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:52,788 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:52,790 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:52,791 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:52,792 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:53,384 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:53,792 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:53,923 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:53,926 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:53,926 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:53,927 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:54,385 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:54,385 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:55,058 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:55,061 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:55,061 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:55,062 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:55,386 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:56,192 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:56,194 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:56,195 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:56,196 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:56,387 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:56,387 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:57,423 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:57,425 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:57,426 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:57,427 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:58,389 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:43:58,389 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:58,564 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:58,566 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:58,567 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:58,568 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:43:58,948 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:43:58,948 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:43:58,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:43:58,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:43:58,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:43:59,390 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:43:59,699 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:43:59,701 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:43:59,701 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:43:59,702 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:00,391 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:00,391 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:00,833 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:00,835 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:00,835 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:00,836 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:01,392 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:01,973 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:01,975 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:01,976 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:01,977 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:02,393 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:02,393 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:03,108 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:03,112 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:03,112 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:03,113 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:03,394 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:04,113 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:04,241 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:04,245 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:04,245 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:04,246 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:04,395 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:04,395 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:05,379 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:05,382 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:05,382 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:05,383 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:05,396 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:06,396 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:06,522 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:06,524 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:06,524 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:06,526 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:07,397 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:07,670 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:07,672 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:07,672 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:07,673 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:08,398 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:08,398 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:08,807 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:08,810 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:08,810 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:08,811 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:09,399 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:09,812 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:09,967 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:09,969 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:09,969 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:09,970 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:10,400 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:10,400 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:11,095 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:11,097 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:11,097 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:11,099 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:11,401 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:12,230 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:12,232 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:12,232 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:12,233 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:12,402 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:12,402 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:13,366 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:13,368 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:13,369 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:13,370 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:13,403 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:13,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:44:13,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:44:13,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:44:14,404 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:14,666 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:14,668 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:14,668 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:14,669 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:15,405 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:15,707 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:15,802 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:15,804 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:15,805 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:15,806 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:16,406 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:16,406 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:16,934 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:16,936 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:16,936 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:16,938 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:17,407 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:18,070 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:18,072 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:18,073 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:18,074 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:18,408 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:18,408 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:19,205 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:19,209 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:19,210 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:19,211 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:19,409 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:20,333 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:20,335 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:20,336 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:20,337 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:20,410 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:20,410 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:21,337 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:21,467 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:21,469 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:21,470 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:21,471 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:22,411 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:22,411 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:22,590 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:22,592 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:22,592 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:22,593 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:23,412 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:23,724 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:23,726 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:23,727 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:23,728 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:24,413 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:24,413 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:24,853 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:24,856 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:24,856 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:24,857 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:25,414 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:25,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:25,982 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:25,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:25,984 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:26,415 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:26,415 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:26,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:27,105 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:27,107 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:27,107 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:27,109 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:27,416 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:28,225 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:28,227 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:28,228 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:28,229 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:28,417 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:28,417 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:28,949 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:44:28,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:44:28,979 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:44:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:44:29,356 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:29,357 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:29,357 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:29,358 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:29,418 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:30,419 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:30,575 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:30,577 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:30,578 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:30,579 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:31,420 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:31,698 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:31,700 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:31,700 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:31,702 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:32,421 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:32,421 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:32,702 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:32,823 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:32,825 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:32,825 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:32,827 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:33,422 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:33,953 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:33,956 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:33,956 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:33,957 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:34,423 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:34,423 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:35,092 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:35,095 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:35,095 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:35,096 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:35,424 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:36,219 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:36,221 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:36,222 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:36,223 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:36,425 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:36,426 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:37,347 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:37,350 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:37,350 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:37,351 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:37,427 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:38,352 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:38,427 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:38,479 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:38,482 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:38,482 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:38,483 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:39,428 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:39,684 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:39,686 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:39,686 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:39,687 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:40,429 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:40,430 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:40,806 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:40,809 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:40,809 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:40,810 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:41,430 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:41,939 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:41,942 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:41,942 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:41,943 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:42,432 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:42,432 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:43,076 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:43,079 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:43,079 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:43,080 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:43,433 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:43,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:44:43,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:44:43,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:44:44,211 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:44,251 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:44,252 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:44,252 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:44,255 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:44,434 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:44,434 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:45,345 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:45,347 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:45,347 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:45,348 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:45,435 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:46,435 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:46,478 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:46,480 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:46,481 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:46,482 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:47,437 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:47,693 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:47,696 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:47,696 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:47,697 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:48,438 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:48,438 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:48,823 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:48,825 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:48,826 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:48,827 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:49,439 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:49,827 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:49,968 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:49,970 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:49,971 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:49,972 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:50,440 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:50,440 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:51,107 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:51,110 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:51,110 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:51,111 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:51,441 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:52,241 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:52,243 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:52,243 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:52,244 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:52,442 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:52,442 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:53,367 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:53,369 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:53,370 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:53,371 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:53,443 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:54,443 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:54,501 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:54,504 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:54,504 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:54,505 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:55,444 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:55,505 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:44:55,637 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:55,639 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:55,640 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:55,641 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:56,445 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:56,445 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:56,776 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:56,779 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:56,779 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:56,780 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:57,446 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:57,912 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:57,914 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:57,914 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:57,915 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:58,447 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:44:58,448 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:44:58,949 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:44:58,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:44:58,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:44:58,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:44:59,049 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:44:59,191 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:44:59,192 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:44:59,193 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:44:59,448 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:00,185 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:00,188 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:00,188 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:00,189 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:00,450 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:00,450 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:01,190 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:01,314 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:01,316 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:01,316 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:01,317 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:01,451 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:02,452 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:02,455 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:02,457 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:02,457 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:02,459 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:03,453 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:03,592 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:03,594 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:03,595 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:03,596 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:04,454 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:04,454 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:04,825 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:04,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:04,828 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:04,829 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:05,455 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:05,963 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:05,966 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:05,966 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:05,967 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:06,456 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:06,456 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:06,968 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:07,103 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:07,105 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:07,106 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:07,107 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:07,457 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:08,239 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:08,242 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:08,242 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:08,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:08,458 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:08,459 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:09,371 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:09,374 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:09,374 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:09,375 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:09,459 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:10,460 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:10,507 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:10,510 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:10,510 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:10,511 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:11,461 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:11,649 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:11,651 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:11,651 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:11,652 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:12,462 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:12,462 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:12,653 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:12,785 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:12,787 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:12,788 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:12,789 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:13,463 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:13,928 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:13,930 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:13,930 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:13,932 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:13,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:45:13,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:45:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:45:14,464 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:14,464 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:15,067 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:15,068 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:15,069 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:15,070 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:15,465 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:16,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:16,210 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:16,210 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:16,212 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:16,466 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:16,466 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:17,355 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:17,357 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:17,357 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:17,359 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:17,467 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:18,359 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:18,467 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:18,495 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:18,497 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:18,497 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:18,498 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:19,468 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:19,631 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:19,634 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:19,634 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:19,636 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:20,469 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:20,469 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:20,770 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:20,772 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:20,773 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:20,774 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:21,470 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:22,002 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:22,004 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:22,005 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:22,006 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:22,471 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:22,472 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:23,145 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:23,148 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:23,148 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:23,149 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:23,472 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:24,150 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:24,280 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:24,306 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:24,306 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:24,307 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:24,473 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:24,473 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:25,419 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:25,422 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:25,423 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:25,424 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:25,474 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:26,475 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:26,554 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:26,556 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:26,556 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:26,557 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:27,476 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:27,691 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:27,693 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:27,693 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:27,694 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:28,477 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:28,477 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:28,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:28,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:28,828 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:28,830 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:28,950 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:45:28,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:45:28,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:45:28,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:45:29,234 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:29,478 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:29,961 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:29,962 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:29,963 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:29,964 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:30,479 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:30,479 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:31,095 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:31,098 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:31,098 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:31,099 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:31,480 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:32,228 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:32,230 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:32,231 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:32,232 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:32,481 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:32,481 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:33,360 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:33,362 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:33,362 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:33,364 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:33,482 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:34,364 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:34,482 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:34,507 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:34,509 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:34,510 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:34,511 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:35,483 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:35,656 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:35,658 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:35,659 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:35,660 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:36,484 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:36,484 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:36,799 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:36,802 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:36,802 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:36,803 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:37,485 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:38,032 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:38,034 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:38,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:38,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:38,486 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:38,487 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:39,172 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:39,175 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:39,175 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:39,176 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:39,487 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:40,177 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:40,309 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:40,311 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:40,311 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:40,313 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:40,488 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:40,488 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:41,447 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:41,450 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:41,450 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:41,451 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:41,489 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:42,490 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:42,588 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:42,591 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:42,591 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:42,592 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:43,490 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:43,723 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:43,725 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:43,725 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:43,726 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:43,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:45:43,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:45:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:45:44,492 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:44,492 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:44,867 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:44,868 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:44,868 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:44,870 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:45,493 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:45,918 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:46,051 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:46,054 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:46,054 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:46,055 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:46,494 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:46,494 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:47,204 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:47,206 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:47,206 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:47,207 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:47,494 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:48,350 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:48,353 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:48,353 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:48,354 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:48,495 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:48,496 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:49,499 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:49,502 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:49,502 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:49,503 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:50,497 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:50,497 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:50,644 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:50,646 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:50,646 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:50,648 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:51,498 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:51,648 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:51,786 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:51,788 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:51,788 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:51,789 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:52,499 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:52,499 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:52,931 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:52,933 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:52,933 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:52,935 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:53,500 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:54,070 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:54,072 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:54,073 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:54,074 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:54,501 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:54,501 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:55,210 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:55,212 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:55,213 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:55,214 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:55,502 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:56,445 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:56,449 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:56,449 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:56,450 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:56,504 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:56,504 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:57,451 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:45:57,591 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:57,593 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:57,594 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:57,595 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:58,506 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:45:58,506 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:58,763 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:58,765 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:58,766 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:58,767 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:45:58,951 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:45:58,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:45:58,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:45:58,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:45:59,507 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:45:59,941 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:45:59,943 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:45:59,943 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:45:59,944 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:00,508 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:00,508 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:01,081 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:01,084 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:01,084 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:01,086 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:01,509 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:02,222 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:02,225 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:02,225 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:02,226 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:02,510 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:02,510 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:03,227 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:03,391 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:03,393 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:03,394 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:03,395 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:03,511 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:04,511 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:04,533 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:04,535 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:04,536 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:04,537 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:04,639 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:05,512 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:05,512 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:05,950 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:05,952 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:05,953 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:05,954 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:06,513 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:06,513 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:07,095 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:07,098 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:07,098 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:07,099 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:07,514 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:08,242 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:08,244 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:08,244 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:08,245 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:08,246 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:08,515 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:08,515 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:09,391 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:09,393 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:09,394 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:09,394 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:09,516 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:10,517 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:10,531 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:10,533 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:10,534 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:10,535 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:11,518 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:11,672 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:11,675 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:11,675 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:11,676 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:12,519 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:12,519 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:12,911 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:12,913 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:12,914 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:12,915 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:13,520 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:13,915 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:13,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:46:13,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:46:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:46:14,052 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:14,224 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:14,224 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:14,225 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:14,521 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:14,521 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:15,190 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:15,192 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:15,193 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:15,194 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:15,522 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:16,335 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:16,337 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:16,337 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:16,338 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:16,523 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:16,523 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:17,481 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:17,483 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:17,483 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:17,484 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:17,524 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:18,525 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:18,628 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:18,630 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:18,630 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:18,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:19,525 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:19,632 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:19,774 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:19,776 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:19,777 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:19,778 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:20,526 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:20,527 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:20,922 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:20,924 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:20,924 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:20,926 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:21,527 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:22,081 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:22,084 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:22,084 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:22,085 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:22,528 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:22,529 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:23,228 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:23,230 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:23,231 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:23,232 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:23,529 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:24,368 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:24,370 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:24,371 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:24,372 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:24,530 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:24,531 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:25,372 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:25,510 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:25,514 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:25,514 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:25,515 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:25,531 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:26,532 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:26,645 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:26,647 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:26,648 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:26,649 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:27,533 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:27,779 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:27,782 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:27,782 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:27,783 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:28,534 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:28,534 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:28,915 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:28,917 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:28,917 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:28,918 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:28,952 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:46:28,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:46:28,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:46:28,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:46:29,535 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:30,139 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:30,141 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:30,141 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:30,142 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:30,536 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:30,536 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:31,183 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:31,268 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:31,270 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:31,270 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:31,271 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:31,537 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:32,396 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:32,398 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:32,398 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:32,399 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:32,538 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:32,538 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:33,527 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:33,529 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:33,529 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:33,530 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:33,539 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:34,539 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:34,663 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:34,665 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:34,666 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:34,667 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:35,540 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:35,799 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:35,801 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:35,801 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:35,803 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:36,541 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:36,541 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:36,803 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:36,929 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:36,931 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:36,932 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:36,933 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:37,543 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:38,067 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:38,069 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:38,069 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:38,070 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:38,544 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:38,544 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:39,203 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:39,205 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:39,205 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:39,206 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:39,545 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:40,337 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:40,339 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:40,339 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:40,340 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:40,546 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:40,546 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:41,470 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:41,472 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:41,472 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:41,473 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:41,547 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:42,474 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:42,547 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:42,607 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:42,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:42,610 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:42,611 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:43,548 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:43,749 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:43,751 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:43,751 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:43,753 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:46:43,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:46:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:46:44,549 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:44,549 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:44,891 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:44,893 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:44,893 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:44,894 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:45,550 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:46,117 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:46,119 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:46,120 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:46,121 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:46,551 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:46,552 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:47,250 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:47,252 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:47,252 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:47,253 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:47,552 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:48,254 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:48,383 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:48,385 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:48,385 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:48,387 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:48,553 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:48,554 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:49,514 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:49,516 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:49,516 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:49,518 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:49,554 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:50,555 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:50,656 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:50,658 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:50,658 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:50,659 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:51,556 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:51,786 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:51,788 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:51,789 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:51,790 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:52,557 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:52,557 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:52,925 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:52,928 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:52,928 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:52,929 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:53,558 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:53,930 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:54,075 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:54,077 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:54,077 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:54,078 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:54,559 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:54,559 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:55,208 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:55,210 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:55,211 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:55,212 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:55,560 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:56,344 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:56,346 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:56,346 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:56,348 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:56,561 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:56,561 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:57,472 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:57,475 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:57,475 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:57,476 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:57,562 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:58,562 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:46:58,606 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:58,608 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:58,609 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:58,610 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:46:58,952 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:46:58,953 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:46:58,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:46:58,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:46:58,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:46:59,563 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:46:59,745 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:46:59,747 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:46:59,747 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:46:59,748 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:00,564 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:00,565 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:00,869 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:00,871 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:00,872 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:00,873 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:01,565 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:01,997 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:02,000 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:02,000 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:02,002 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:02,567 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:02,567 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:03,220 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:03,223 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:03,224 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:03,225 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:03,568 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:04,225 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:04,349 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:04,351 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:04,352 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:04,353 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:04,569 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:04,569 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:05,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:05,487 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:05,487 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:05,488 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:05,571 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:06,571 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:06,609 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:06,611 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:06,611 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:06,613 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:07,572 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:07,735 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:07,737 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:07,737 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:07,738 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:08,573 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:08,573 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:08,855 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:08,857 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:08,858 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:08,859 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:09,574 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:09,859 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:09,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:09,980 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:09,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:09,981 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:10,575 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:10,575 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:11,113 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:11,115 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:11,115 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:11,116 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:11,576 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:12,246 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:12,248 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:12,249 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:12,250 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:12,577 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:12,577 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:13,377 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:13,379 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:13,380 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:13,381 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:13,578 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:13,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:47:13,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:47:13,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:47:14,501 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:14,503 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:14,503 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:14,504 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:14,579 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:14,579 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:15,543 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:15,623 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:15,625 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:15,625 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:15,626 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:16,581 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:16,581 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:16,741 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:16,743 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:16,743 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:16,744 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:17,582 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:17,860 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:17,887 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:17,887 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:17,888 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:18,583 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:18,583 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:19,073 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:19,076 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:19,076 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:19,077 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:19,584 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:20,192 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:20,195 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:20,195 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:20,196 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:20,585 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:20,585 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:21,197 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:21,310 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:21,313 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:21,313 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:21,314 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:21,586 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:22,429 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:22,431 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:22,431 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:22,433 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:22,587 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:22,587 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:23,548 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:23,550 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:23,551 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:23,552 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:23,588 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:24,588 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:24,669 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:24,671 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:24,671 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:24,672 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:25,589 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:25,790 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:25,793 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:25,793 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:25,794 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:26,590 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:26,590 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:26,795 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:26,915 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:26,917 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:26,917 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:26,918 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:27,591 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:28,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:28,040 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:28,040 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:28,042 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:28,592 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:28,592 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:28,953 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:47:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:47:28,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:47:28,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:47:29,159 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:29,246 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:29,246 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:29,247 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:29,593 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:30,279 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:30,281 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:30,282 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:30,283 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:30,594 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:30,594 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:31,399 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:31,401 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:31,401 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:31,402 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:31,595 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:32,403 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:32,517 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:32,519 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:32,520 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:32,521 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:32,596 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:32,596 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:33,638 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:33,640 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:33,640 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:33,642 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:34,597 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:34,598 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:34,759 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:34,761 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:34,762 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:34,763 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:35,599 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:35,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:35,985 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:35,986 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:35,987 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:36,600 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:36,600 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:37,107 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:37,110 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:37,110 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:37,111 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:37,601 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:38,112 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:38,230 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:38,232 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:38,233 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:38,234 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:38,602 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:38,602 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:39,354 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:39,356 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:39,357 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:39,358 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:39,603 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:40,477 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:40,480 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:40,480 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:40,481 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:40,604 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:40,604 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:41,600 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:41,602 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:41,602 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:41,603 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:41,605 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:42,605 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:42,726 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:42,728 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:42,728 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:42,729 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:43,606 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:43,730 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:43,869 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:43,871 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:43,872 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:43,873 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:47:43,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:47:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:47:44,607 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:44,607 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:44,999 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:45,001 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:45,001 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:45,002 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:45,608 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:46,129 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:46,132 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:46,133 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:46,134 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:46,609 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:46,610 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:47,258 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:47,260 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:47,260 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:47,262 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:47,610 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:48,386 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:48,388 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:48,389 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:48,390 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:48,611 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:48,612 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:49,391 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:49,514 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:49,517 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:49,517 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:49,518 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:49,612 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:50,613 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:50,643 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:50,645 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:50,646 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:50,647 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:51,614 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:51,757 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:51,759 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:51,760 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:51,761 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:52,616 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:52,617 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:52,962 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:52,965 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:52,965 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:52,967 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:53,618 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:54,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:54,074 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:54,075 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:54,076 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:54,619 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:54,620 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:55,076 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:47:55,186 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:55,188 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:55,188 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:55,189 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:55,621 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:56,299 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:56,301 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:56,301 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:56,302 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:56,622 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:56,622 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:57,410 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:57,412 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:57,412 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:57,414 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:57,623 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:58,525 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:58,527 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:58,527 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:58,528 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:47:58,626 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:47:58,626 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:47:58,954 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:47:58,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:47:58,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:47:58,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:47:59,649 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:47:59,650 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:47:59,650 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:47:59,651 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:00,628 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:00,628 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:00,691 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:00,765 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:00,767 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:00,767 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:00,769 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:01,629 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:01,875 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:01,877 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:01,877 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:01,879 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:02,631 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:02,631 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:02,996 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:02,999 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:02,999 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:03,000 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:03,632 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:04,166 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:04,168 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:04,168 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:04,169 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:04,634 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:04,634 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:05,280 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:05,282 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:05,282 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:05,284 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:05,635 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:06,284 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:06,396 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:06,398 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:06,398 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:06,399 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:06,637 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:06,637 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:07,513 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:07,515 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:07,516 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:07,517 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:07,637 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:08,625 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:08,627 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:08,627 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:08,629 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:08,639 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:08,639 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:09,839 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:09,842 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:09,842 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:09,843 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:10,641 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:10,967 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:10,969 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:10,970 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:10,971 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:11,642 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:11,642 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:11,971 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:12,167 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:12,169 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:12,169 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:12,170 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:12,643 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:13,299 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:13,301 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:13,301 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:13,303 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:13,644 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:13,644 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:13,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:48:13,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:48:13,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:48:14,430 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:14,432 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:14,432 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:14,433 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:14,645 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:15,563 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:15,565 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:15,565 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:15,566 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:15,645 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:15,646 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:16,696 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:16,698 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:16,699 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:16,700 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:17,647 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:17,647 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:17,700 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:17,830 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:17,832 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:17,832 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:17,833 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:18,648 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:18,950 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:18,952 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:18,953 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:18,954 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:19,649 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:19,649 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:20,073 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:20,075 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:20,075 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:20,077 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:20,650 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:21,202 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:21,204 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:21,205 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:21,206 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:21,651 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:21,651 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:22,335 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:22,337 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:22,337 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:22,338 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:22,652 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:23,339 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:23,469 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:23,472 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:23,472 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:23,473 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:23,653 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:23,653 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:24,597 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:24,599 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:24,599 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:24,600 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:24,654 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:25,654 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:25,727 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:25,729 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:25,730 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:25,731 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:26,655 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:26,856 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:26,860 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:26,861 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:26,862 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:27,657 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:27,657 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:28,079 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:28,082 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:28,082 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:28,083 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:28,658 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:28,955 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:48:28,955 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:48:28,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:48:28,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:48:29,210 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:29,212 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:29,212 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:29,213 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:29,659 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:29,659 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:30,336 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:30,339 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:30,339 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:30,340 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:30,660 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:31,462 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:31,464 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:31,465 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:31,466 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:31,662 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:31,662 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:32,595 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:32,598 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:32,598 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:32,599 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:32,663 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:33,663 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:33,733 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:33,735 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:33,736 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:33,737 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:34,665 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:34,737 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:34,868 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:34,870 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:34,871 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:34,872 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:35,666 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:35,666 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:35,993 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:35,995 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:35,995 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:35,996 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:36,667 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:37,117 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:37,119 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:37,119 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:37,120 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:37,668 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:37,668 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:38,249 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:38,251 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:38,252 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:38,253 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:38,670 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:39,363 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:39,366 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:39,366 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:39,367 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:39,671 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:39,671 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:40,367 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:40,472 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:40,474 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:40,474 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:40,475 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:40,672 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:41,587 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:41,589 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:41,590 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:41,591 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:41,673 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:41,674 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:42,715 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:42,717 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:42,718 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:42,719 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:43,675 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:43,675 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:43,830 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:43,832 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:43,833 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:43,834 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:48:43,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:48:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:48:44,677 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:45,042 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:45,044 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:45,044 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:45,045 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:45,678 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:45,679 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:46,087 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:46,168 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:46,170 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:46,170 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:46,172 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:46,680 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:47,286 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:47,289 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:47,289 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:47,290 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:47,681 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:47,681 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:48,401 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:48,403 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:48,404 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:48,405 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:48,682 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:49,524 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:49,527 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:49,527 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:49,528 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:49,683 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:49,683 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:50,644 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:50,646 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:50,647 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:50,648 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:50,684 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:51,648 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:51,685 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:51,760 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:51,762 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:51,763 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:51,764 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:52,686 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:52,875 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:52,878 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:52,878 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:52,879 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:53,687 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:53,687 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:53,990 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:53,992 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:53,993 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:53,994 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:54,688 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:55,108 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:55,111 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:55,111 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:55,112 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:55,689 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:55,689 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:56,222 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:56,225 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:56,225 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:56,226 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:56,690 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:57,227 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:48:57,337 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:57,340 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:57,340 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:57,341 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:57,691 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:57,692 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:58,457 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:58,460 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:58,460 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:58,461 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:58,692 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:48:58,956 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:48:58,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:48:58,981 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:48:58,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:48:59,585 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:48:59,586 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:48:59,587 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:48:59,588 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:48:59,693 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:48:59,694 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:00,718 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:00,720 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:00,721 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:00,722 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:01,695 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:01,695 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:01,929 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:01,931 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:01,931 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:01,932 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:02,696 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:02,933 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:03,050 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:03,052 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:03,053 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:03,054 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:03,697 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:03,697 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:04,170 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:04,173 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:04,173 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:04,174 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:04,698 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:05,291 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:05,293 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:05,293 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:05,294 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:05,699 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:05,699 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:06,413 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:06,416 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:06,416 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:06,418 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:06,700 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:07,535 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:07,538 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:07,538 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:07,539 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:07,701 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:07,701 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:08,540 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:08,659 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:08,661 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:08,662 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:08,663 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:08,702 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:09,703 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:09,782 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:09,784 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:09,784 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:09,786 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:10,704 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:10,904 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:10,906 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:10,907 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:10,908 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:11,705 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:11,705 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:12,034 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:12,037 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:12,037 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:12,038 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:12,706 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:13,165 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:13,167 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:13,167 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:13,168 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:13,706 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:13,707 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:13,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:49:13,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:49:13,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:49:14,246 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:14,295 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:14,297 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:14,297 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:14,298 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:14,707 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:15,436 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:15,438 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:15,438 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:15,439 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:15,708 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:15,708 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:16,568 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:16,570 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:16,570 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:16,571 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:16,709 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:17,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:17,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:17,706 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:17,707 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:17,710 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:17,710 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:18,921 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:18,923 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:18,923 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:18,924 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:19,712 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:19,712 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:19,925 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:20,040 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:20,042 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:20,042 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:20,044 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:20,713 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:21,166 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:21,192 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:21,193 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:21,194 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:21,715 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:21,715 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:22,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:22,295 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:22,295 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:22,299 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:22,716 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:23,423 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:23,425 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:23,425 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:23,426 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:23,717 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:23,717 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:24,549 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:24,551 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:24,552 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:24,553 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:24,718 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:25,553 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:25,671 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:25,673 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:25,673 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:25,674 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:25,720 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:25,720 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:26,793 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:26,795 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:26,795 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:26,796 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:27,721 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:27,721 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:27,947 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:27,949 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:27,949 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:27,950 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:28,723 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:28,956 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:49:28,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:49:28,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:49:28,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:49:29,066 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:29,207 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:29,208 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:29,209 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:29,724 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:29,724 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:30,186 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:30,188 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:30,189 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:30,190 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:30,725 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:31,190 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:31,308 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:31,310 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:31,311 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:31,312 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:31,726 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:31,726 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:32,427 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:32,429 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:32,429 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:32,430 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:32,727 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:33,545 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:33,547 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:33,547 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:33,548 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:33,729 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:33,729 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:34,757 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:34,759 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:34,760 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:34,761 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:35,730 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:35,731 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:35,879 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:35,881 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:35,881 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:35,882 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:36,732 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:36,883 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:36,995 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:36,997 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:36,998 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:36,999 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:37,733 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:37,733 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:38,117 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:38,119 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:38,120 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:38,121 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:38,734 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:39,239 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:39,242 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:39,242 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:39,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:39,735 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:39,735 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:40,359 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:40,362 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:40,362 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:40,363 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:40,737 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:41,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:41,486 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:41,486 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:41,487 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:41,738 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:41,738 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:42,488 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:42,619 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:42,621 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:42,622 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:42,623 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:42,739 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:43,740 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:43,743 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:43,745 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:43,745 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:43,747 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:43,981 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:49:43,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:49:43,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:49:44,741 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:44,874 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:44,876 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:44,876 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:44,877 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:45,742 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:45,742 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:46,001 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:46,004 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:46,004 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:46,005 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:46,743 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:47,123 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:47,141 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:47,141 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:47,142 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:47,744 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:47,744 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:48,143 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:48,250 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:48,252 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:48,253 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:48,255 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:48,745 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:49,384 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:49,386 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:49,387 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:49,388 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:49,747 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:49,747 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:50,608 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:50,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:50,611 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:50,612 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:50,748 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:51,749 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:51,762 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:51,764 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:51,764 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:51,765 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:51,898 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:52,754 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:53,206 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:53,208 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:53,209 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:53,209 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:53,210 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:53,756 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:53,756 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:54,330 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:54,333 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:54,333 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:54,334 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:54,757 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:55,449 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:55,451 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:55,452 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:55,453 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:55,758 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:55,758 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:56,589 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:56,591 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:56,592 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:56,593 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:56,759 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:57,727 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:57,729 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:57,729 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:57,730 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:57,760 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:57,760 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:49:58,731 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:49:58,871 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:49:58,873 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:49:58,873 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:49:58,874 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:49:58,957 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:49:58,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:49:58,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:49:58,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:49:59,761 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:49:59,761 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:00,017 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:00,019 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:00,019 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:00,020 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:00,762 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:01,161 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:01,163 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:01,163 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:01,165 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:01,764 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:01,764 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:02,300 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:02,303 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:02,303 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:02,304 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:02,765 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:03,442 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:03,444 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:03,444 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:03,445 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:03,766 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:03,766 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:04,446 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:04,578 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:04,580 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:04,581 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:04,582 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:04,767 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:05,717 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:05,721 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:05,721 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:05,722 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:05,768 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:05,768 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:06,857 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:06,859 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:06,859 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:06,860 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:07,769 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:07,770 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:08,087 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:08,090 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:08,090 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:08,091 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:08,770 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:09,226 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:09,228 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:09,229 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:09,230 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:09,772 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:09,772 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:10,230 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:10,361 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:10,363 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:10,364 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:10,365 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:10,773 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:11,498 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:11,500 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:11,500 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:11,501 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:11,774 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:11,774 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:12,633 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:12,636 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:12,636 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:12,637 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:12,775 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:13,770 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:13,772 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:13,772 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:13,773 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:13,776 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:13,776 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:50:13,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:50:13,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:50:14,908 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:14,909 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:14,909 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:14,911 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:15,777 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:15,777 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:15,951 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:16,042 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:16,045 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:16,045 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:16,046 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:16,778 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:17,178 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:17,181 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:17,181 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:17,182 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:17,779 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:17,779 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:18,315 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:18,317 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:18,318 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:18,319 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:18,780 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:19,451 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:19,453 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:19,454 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:19,455 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:19,781 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:19,781 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:20,588 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:20,590 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:20,591 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:20,592 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:20,782 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:21,592 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:21,725 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:21,727 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:21,728 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:21,729 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:21,783 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:21,783 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:22,865 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:22,868 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:22,868 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:22,869 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:23,785 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:23,785 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:24,005 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:24,007 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:24,008 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:24,009 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:24,786 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:25,238 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:25,240 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:25,240 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:25,241 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:25,787 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:25,787 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:26,380 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:26,383 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:26,384 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:26,385 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:26,788 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:27,385 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:27,521 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:27,524 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:27,524 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:27,525 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:27,789 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:27,789 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:28,659 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:28,662 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:28,662 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:28,663 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:28,790 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:28,958 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:50:28,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:50:28,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:50:28,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:50:29,790 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:29,802 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:29,804 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:29,804 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:29,805 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:30,791 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:30,948 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:30,950 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:30,950 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:30,952 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:31,792 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:31,792 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:32,099 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:32,102 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:32,102 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:32,103 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:32,793 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:33,103 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:33,261 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:33,264 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:33,264 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:33,265 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:33,794 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:33,794 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:34,404 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:34,407 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:34,407 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:34,408 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:34,795 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:35,549 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:35,552 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:35,552 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:35,553 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:35,796 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:35,796 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:36,690 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:36,692 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:36,692 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:36,694 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:36,797 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:37,798 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:37,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:37,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:37,829 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:37,830 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:38,799 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:38,830 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:38,961 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:38,963 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:38,963 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:38,964 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:39,800 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:39,800 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:40,088 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:40,091 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:40,091 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:40,092 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:40,801 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:41,221 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:41,223 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:41,224 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:41,225 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:41,802 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:41,802 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:42,460 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:42,462 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:42,463 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:42,464 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:42,803 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:43,607 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:43,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:43,610 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:43,611 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:43,804 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:43,804 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:43,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:50:43,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:50:43,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:50:44,251 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:44,754 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:44,756 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:44,756 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:44,757 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:44,805 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:45,806 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:45,891 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:45,893 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:45,893 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:45,894 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:46,807 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:47,028 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:47,031 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:47,031 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:47,032 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:47,808 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:47,808 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:48,157 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:48,159 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:48,160 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:48,161 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:48,809 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:49,300 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:49,302 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:49,302 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:49,303 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:49,303 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:49,810 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:49,810 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:50,438 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:50,440 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:50,441 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:50,442 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:50,811 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:51,586 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:51,588 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:51,589 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:51,590 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:51,812 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:51,813 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:52,723 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:52,726 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:52,726 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:52,727 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:52,813 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:53,814 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:53,859 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:53,861 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:53,861 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:53,862 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:54,815 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:54,863 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:50:54,996 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:54,998 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:54,998 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:54,999 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:55,816 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:55,816 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:56,124 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:56,127 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:56,127 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:56,128 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:56,817 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:57,264 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:57,265 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:57,266 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:57,267 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:57,818 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:57,818 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:58,396 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:58,398 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:58,398 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:58,400 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:58,819 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:50:58,959 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:50:58,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:50:58,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:50:58,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:50:59,628 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:50:59,630 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:50:59,630 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:50:59,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:50:59,820 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:50:59,820 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:00,671 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:00,762 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:00,765 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:00,765 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:00,766 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:00,821 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:01,822 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:01,906 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:01,908 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:01,908 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:01,910 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:02,823 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:03,050 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:03,052 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:03,052 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:03,053 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:03,824 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:03,824 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:04,188 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:04,191 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:04,191 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:04,192 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:04,825 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:05,323 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:05,325 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:05,326 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:05,327 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:05,826 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:05,826 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:06,327 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:06,440 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:06,442 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:06,442 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:06,443 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:06,827 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:07,555 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:07,557 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:07,558 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:07,559 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:07,828 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:07,828 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:08,681 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:08,683 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:08,684 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:08,685 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:08,829 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:09,802 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:09,805 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:09,805 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:09,806 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:09,830 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:09,830 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:10,923 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:10,925 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:10,925 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:10,926 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:11,831 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:11,832 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:11,927 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:12,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:12,040 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:12,040 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:12,041 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:12,832 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:13,147 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:13,149 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:13,149 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:13,150 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:13,833 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:13,833 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:51:13,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:51:13,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:51:14,256 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:14,257 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:14,258 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:14,259 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:14,834 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:15,364 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:15,366 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:15,367 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:15,368 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:15,835 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:15,835 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:16,568 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:16,571 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:16,571 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:16,572 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:16,836 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:17,573 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:17,684 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:17,686 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:17,687 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:17,688 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:17,837 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:17,837 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:18,796 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:18,822 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:18,822 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:18,823 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:18,838 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:19,839 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:19,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:19,913 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:19,913 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:19,914 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:20,840 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:21,030 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:21,032 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:21,033 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:21,034 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:21,841 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:21,841 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:22,138 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:22,140 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:22,141 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:22,142 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:22,842 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:23,142 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:23,246 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:23,248 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:23,248 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:23,249 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:23,842 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:23,843 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:24,354 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:24,356 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:24,357 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:24,358 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:24,843 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:25,472 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:25,474 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:25,474 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:25,475 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:25,844 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:25,845 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:26,588 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:26,590 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:26,591 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:26,592 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:26,845 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:27,708 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:27,713 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:27,713 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:27,714 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:27,846 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:27,846 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:28,715 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:28,822 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:28,824 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:28,824 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:28,825 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:28,847 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:28,959 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:51:28,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:51:28,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:51:28,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:51:29,848 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:29,942 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:29,944 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:29,944 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:29,945 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:30,849 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:31,065 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:31,067 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:31,067 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:31,068 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:31,850 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:31,850 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:32,194 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:32,197 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:32,198 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:32,199 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:32,851 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:33,420 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:33,423 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:33,423 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:33,424 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:33,852 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:33,852 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:34,425 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:34,547 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:34,549 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:34,549 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:34,551 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:34,853 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:35,677 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:35,679 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:35,679 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:35,680 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:35,854 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:35,854 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:36,798 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:36,800 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:36,801 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:36,802 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:36,855 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:37,855 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:37,918 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:37,920 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:37,921 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:37,922 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:38,856 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:39,045 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:39,047 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:39,047 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:39,048 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:39,857 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:39,857 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:40,049 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:40,171 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:40,173 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:40,173 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:40,174 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:40,858 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:41,299 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:41,301 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:41,302 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:41,303 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:41,859 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:41,859 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:42,419 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:42,421 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:42,422 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:42,423 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:42,860 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:43,538 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:43,540 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:43,541 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:43,542 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:43,861 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:43,861 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:43,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:51:43,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:51:43,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:51:44,662 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:44,664 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:44,664 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:44,665 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:44,862 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:45,703 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:45,774 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:45,776 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:45,776 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:45,777 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:45,863 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:45,863 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:46,884 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:46,886 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:46,886 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:46,888 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:47,865 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:47,865 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:48,002 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:48,004 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:48,005 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:48,006 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:48,865 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:49,123 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:49,125 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:49,125 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:49,126 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:49,867 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:49,867 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:50,343 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:50,345 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:50,345 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:50,347 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:50,868 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:51,347 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:51,464 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:51,467 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:51,467 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:51,468 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:51,869 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:51,869 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:52,581 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:52,583 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:52,584 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:52,585 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:52,870 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:53,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:53,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:53,707 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:53,708 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:53,871 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:53,871 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:54,828 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:54,830 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:54,830 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:54,831 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:54,872 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:55,872 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:55,959 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:55,961 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:55,961 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:55,962 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:56,873 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:56,964 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:51:57,085 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:57,087 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:57,087 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:57,088 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:57,874 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:57,874 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:58,215 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:58,218 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:58,218 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:58,219 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:58,875 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:51:58,960 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:51:58,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:51:58,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:51:58,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:51:59,350 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:51:59,351 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:51:59,352 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:51:59,353 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:51:59,876 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:51:59,876 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:00,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:00,485 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:00,486 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:00,487 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:00,877 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:01,604 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:01,607 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:01,607 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:01,608 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:01,878 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:01,878 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:02,609 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:02,733 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:02,735 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:02,736 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:02,737 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:02,879 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:03,853 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:03,856 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:03,856 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:03,857 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:03,880 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:03,880 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:04,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:04,987 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:04,988 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:04,989 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:05,882 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:05,882 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:06,115 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:06,118 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:06,118 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:06,119 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:06,883 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:07,392 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:07,395 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:07,395 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:07,396 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:07,884 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:07,884 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:08,397 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:08,516 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:08,518 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:08,518 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:08,519 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:08,885 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:09,649 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:09,651 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:09,651 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:09,652 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:09,886 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:09,886 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:10,771 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:10,773 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:10,774 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:10,775 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:10,887 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:11,888 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:11,905 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:11,907 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:11,907 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:11,908 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:12,889 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:13,036 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:13,038 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:13,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:13,039 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:13,889 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:13,890 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:13,982 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:52:13,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:52:13,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:52:14,155 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:14,162 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:14,163 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:14,163 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:14,164 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:14,890 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:15,285 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:15,287 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:15,288 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:15,289 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:15,891 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:15,892 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:16,404 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:16,406 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:16,407 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:16,408 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:16,892 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:17,524 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:17,526 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:17,526 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:17,527 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:17,893 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:17,894 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:18,644 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:18,646 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:18,647 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:18,648 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:18,894 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:19,648 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:19,765 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:19,768 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:19,768 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:19,769 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:19,895 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:19,895 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:20,883 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:20,885 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:20,886 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:20,887 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:20,896 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:21,897 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:22,003 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:22,005 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:22,006 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:22,007 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:22,898 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:23,215 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:23,218 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:23,218 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:23,219 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:23,899 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:23,899 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:24,337 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:24,339 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:24,340 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:24,341 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:24,900 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:25,341 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:25,458 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:25,460 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:25,461 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:25,462 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:25,901 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:25,901 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:26,578 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:26,580 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:26,581 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:26,582 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:26,902 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:27,694 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:27,697 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:27,697 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:27,698 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:27,903 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:27,904 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:28,807 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:28,810 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:28,810 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:28,811 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:28,904 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:28,961 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:52:28,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:52:28,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:52:28,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:52:29,905 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:29,923 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:29,925 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:29,925 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:29,926 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:30,906 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:30,967 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:31,033 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:31,035 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:31,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:31,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:31,907 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:31,907 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:32,143 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:32,145 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:32,145 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:32,146 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:32,908 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:33,260 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:33,262 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:33,262 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:33,263 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:33,909 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:33,909 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:34,375 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:34,377 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:34,377 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:34,378 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:34,910 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:35,494 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:35,496 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:35,497 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:35,498 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:35,911 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:35,911 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:36,498 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:36,617 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:36,619 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:36,619 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:36,620 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:36,912 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:37,739 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:37,741 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:37,742 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:37,743 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:37,913 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:37,913 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:38,862 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:38,864 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:38,865 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:38,866 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:38,914 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:39,914 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:40,076 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:40,078 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:40,079 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:40,080 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:40,915 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:41,199 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:41,201 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:41,202 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:41,203 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:41,916 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:41,916 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:42,204 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:42,324 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:42,326 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:42,327 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:42,328 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:42,917 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:43,439 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:43,441 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:43,442 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:43,443 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:43,918 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:43,918 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:52:43,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:52:43,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:52:44,565 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:44,567 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:44,567 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:44,568 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:44,919 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:45,685 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:45,687 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:45,688 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:45,689 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:45,920 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:45,920 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:46,801 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:46,803 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:46,803 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:46,804 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:46,921 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:47,805 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:47,921 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:47,925 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:47,929 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:47,929 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:47,930 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:48,923 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:49,049 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:49,051 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:49,052 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:49,053 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:49,924 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:49,924 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:50,176 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:50,179 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:50,180 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:50,181 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:50,925 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:51,303 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:51,305 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:51,306 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:51,307 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:51,926 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:51,926 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:52,432 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:52,434 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:52,435 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:52,436 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:52,927 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:53,436 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:53,555 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:53,557 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:53,558 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:53,559 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:53,928 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:53,928 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:54,677 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:54,679 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:54,680 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:54,681 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:54,929 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:55,803 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:55,805 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:55,805 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:55,806 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:55,929 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:55,930 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:57,019 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:57,022 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:57,022 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:57,023 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:57,931 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:57,931 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:58,131 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:58,134 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:58,134 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:58,135 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:58,932 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:52:58,962 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:52:58,964 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:52:58,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:52:58,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:52:58,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:52:59,240 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:52:59,242 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:52:59,242 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:52:59,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:52:59,933 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:52:59,933 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:00,350 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:00,352 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:00,353 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:00,354 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:00,934 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:01,478 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:01,480 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:01,481 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:01,482 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:01,935 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:01,936 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:02,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:02,654 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:02,654 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:02,655 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:02,936 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:03,780 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:03,782 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:03,783 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:03,784 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:03,937 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:03,938 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:04,784 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:04,906 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:04,907 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:04,908 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:04,909 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:04,938 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:05,939 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:06,030 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:06,033 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:06,033 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:06,034 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:06,940 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:07,153 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:07,155 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:07,156 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:07,157 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:07,941 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:07,941 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:08,268 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:08,272 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:08,272 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:08,273 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:08,942 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:09,393 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:09,395 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:09,396 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:09,397 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:09,943 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:09,943 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:10,397 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:10,502 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:10,504 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:10,505 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:10,506 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:10,944 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:11,622 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:11,624 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:11,624 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:11,625 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:11,945 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:11,945 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:12,754 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:12,756 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:12,756 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:12,757 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:12,946 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:13,946 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:13,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:13,985 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:13,986 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:13,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:53:13,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:53:13,987 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:13,988 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:53:14,947 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:15,111 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:15,113 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:15,113 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:15,114 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:15,948 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:15,949 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:16,155 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:16,241 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:16,243 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:16,244 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:16,245 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:16,949 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:17,366 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:17,368 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:17,368 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:17,369 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:17,950 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:17,950 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:18,480 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:18,482 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:18,483 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:18,484 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:18,951 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:19,604 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:19,607 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:19,607 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:19,608 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:19,952 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:19,952 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:20,725 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:20,727 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:20,728 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:20,729 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:20,953 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:21,729 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:21,835 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:21,837 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:21,837 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:21,838 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:21,954 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:21,954 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:22,942 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:22,968 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:22,968 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:22,969 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:23,969 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:23,969 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:24,056 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:24,058 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:24,059 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:24,060 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:24,970 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:25,185 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:25,188 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:25,188 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:25,189 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:25,971 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:25,971 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:26,311 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:26,313 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:26,313 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:26,314 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:26,972 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:27,315 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:27,429 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:27,432 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:27,433 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:27,434 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:27,973 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:27,973 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:28,553 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:28,556 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:28,556 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:28,557 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:28,963 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:53:28,974 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:28,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:53:28,983 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:53:28,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:53:29,677 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:29,678 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:29,679 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:29,680 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:29,975 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:29,976 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:30,884 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:30,887 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:30,887 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:30,888 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:30,977 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:31,977 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:32,011 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:32,014 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:32,014 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:32,015 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:32,979 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:33,015 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:33,200 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:33,202 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:33,203 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:33,204 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:33,980 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:33,980 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:34,342 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:34,344 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:34,345 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:34,346 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:34,981 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:35,467 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:35,469 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:35,469 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:35,471 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:35,982 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:35,982 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:36,594 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:36,597 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:36,597 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:36,598 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:36,983 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:37,719 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:37,722 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:37,722 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:37,723 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:37,984 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:37,985 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:38,723 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:38,842 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:38,844 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:38,845 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:38,846 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:38,948 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:38,986 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:39,986 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:40,237 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:40,239 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:40,240 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:40,241 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:40,987 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:41,366 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:41,368 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:41,369 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:41,370 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:41,988 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:41,989 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:42,496 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:42,498 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:42,498 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:42,499 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:42,989 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:43,629 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:43,631 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:43,632 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:43,633 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:43,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:53:43,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:53:43,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:53:43,990 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:43,991 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:44,224 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:44,758 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:44,759 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:44,759 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:44,760 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:44,991 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:45,883 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:45,885 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:45,885 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:45,886 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:45,992 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:45,993 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:47,103 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:47,108 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:47,108 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:47,109 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:47,994 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:47,994 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:48,239 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:48,241 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:48,241 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:48,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:48,995 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:49,243 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:49,370 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:49,372 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:49,372 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:49,373 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:49,996 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:49,996 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:50,493 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:50,495 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:50,496 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:50,497 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:50,997 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:51,614 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:51,616 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:51,616 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:51,618 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:51,998 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:51,998 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:52,743 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:52,745 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:52,745 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:52,746 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:52,999 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:53,872 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:53,874 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:53,875 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:53,876 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:54,000 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:54,000 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:54,877 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:53:55,003 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:55,005 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:55,005 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:55,007 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:56,001 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:56,002 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:56,131 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:56,133 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:56,133 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:56,134 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:57,002 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:57,263 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:57,265 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:57,265 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:57,266 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:58,003 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:53:58,003 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:58,400 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:58,402 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:58,403 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:58,404 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:53:58,964 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:53:58,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:53:58,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:53:58,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:53:59,004 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:53:59,535 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:53:59,537 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:53:59,537 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:53:59,538 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:00,005 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:00,006 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:00,579 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:00,678 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:00,680 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:00,681 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:00,682 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:01,007 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:01,814 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:01,816 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:01,817 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:01,818 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:02,008 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:02,008 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:02,952 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:02,955 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:02,955 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:02,956 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:03,009 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:04,009 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:04,182 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:04,185 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:04,185 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:04,186 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:05,010 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:05,315 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:05,317 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:05,318 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:05,319 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:06,011 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:06,011 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:06,319 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:06,451 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:06,453 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:06,453 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:06,454 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:07,012 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:07,584 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:07,587 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:07,587 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:07,588 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:08,014 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:08,014 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:08,723 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:08,726 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:08,726 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:08,727 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:09,015 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:09,862 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:09,864 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:09,865 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:09,866 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:10,016 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:10,016 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:10,993 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:10,996 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:10,996 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:10,997 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:11,017 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:11,998 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:12,018 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:12,123 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:12,125 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:12,126 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:12,127 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:13,019 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:13,253 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:13,255 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:13,255 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:13,256 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:13,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:54:13,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:54:13,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:54:14,020 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:14,020 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:14,389 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:14,391 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:14,391 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:14,392 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:15,022 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:15,511 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:15,514 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:15,514 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:15,515 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:16,023 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:16,023 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:16,644 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:16,646 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:16,646 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:16,647 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:17,024 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:17,648 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:17,778 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:17,780 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:17,781 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:17,782 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:18,025 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:18,025 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:18,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:18,912 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:18,913 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:18,914 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:19,026 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:20,027 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:20,042 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:20,044 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:20,045 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:20,046 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:21,028 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:21,172 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:21,174 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:21,175 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:21,176 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:22,029 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:22,029 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:22,406 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:22,409 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:22,409 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:22,410 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:23,030 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:23,410 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:23,546 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:23,548 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:23,548 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:23,549 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:24,031 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:24,031 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:24,671 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:24,673 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:24,673 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:24,674 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:25,032 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:25,799 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:25,801 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:25,802 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:25,803 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:26,033 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:26,033 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:26,938 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:26,940 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:26,940 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:26,942 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:27,034 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:28,034 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:28,070 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:28,073 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:28,073 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:28,075 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:28,965 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:54:28,966 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:28,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:54:28,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:54:28,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:54:29,035 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:29,203 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:29,215 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:29,216 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:29,217 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:30,036 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:30,036 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:30,335 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:30,336 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:30,336 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:30,337 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:31,037 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:31,465 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:31,467 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:31,468 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:31,469 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:32,038 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:32,038 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:32,596 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:32,598 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:32,598 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:32,600 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:33,039 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:33,727 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:33,730 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:33,730 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:33,731 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:34,040 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:34,040 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:34,732 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:34,859 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:34,861 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:34,862 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:34,863 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:35,041 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:35,988 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:35,990 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:35,990 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:35,991 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:36,042 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:36,042 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:37,123 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:37,125 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:37,125 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:37,127 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:38,044 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:38,044 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:38,251 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:38,253 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:38,253 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:38,254 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:39,045 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:39,474 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:39,477 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:39,477 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:39,478 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:40,046 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:40,046 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:40,479 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:40,608 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:40,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:40,611 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:40,612 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:41,047 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:41,748 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:41,751 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:41,751 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:41,752 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:42,048 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:42,048 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:42,879 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:42,881 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:42,882 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:42,883 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:43,049 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:43,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:54:43,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:54:43,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:54:44,007 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:44,049 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:44,250 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:44,251 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:44,252 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:45,050 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:45,139 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:45,142 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:45,142 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:45,143 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:46,051 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:46,051 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:46,143 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:46,260 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:46,262 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:46,262 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:46,263 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:47,052 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:47,375 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:47,377 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:47,377 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:47,378 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:48,053 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:48,053 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:48,484 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:48,487 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:48,487 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:48,488 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:49,054 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:49,592 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:49,594 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:49,595 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:49,596 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:50,055 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:50,055 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:50,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:50,706 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:50,706 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:50,707 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:51,056 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:51,708 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:51,816 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:51,818 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:51,819 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:51,820 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:52,057 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:52,057 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:52,935 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:52,937 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:52,938 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:52,939 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:53,058 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:54,058 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:54,061 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:54,063 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:54,063 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:54,064 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:55,059 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:55,238 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:55,240 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:55,240 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:55,241 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:56,061 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:56,061 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:56,446 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:56,449 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:56,449 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:56,450 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:57,062 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:57,450 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:54:57,566 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:57,568 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:57,568 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:57,569 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:58,063 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:54:58,063 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:58,680 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:58,682 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:58,682 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:58,684 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:54:58,965 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:54:58,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:54:58,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:54:58,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:54:59,064 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:54:59,794 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:54:59,795 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:54:59,795 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:54:59,796 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:00,065 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:00,065 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:00,913 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:00,916 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:00,916 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:00,918 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:01,066 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:02,034 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:02,036 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:02,037 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:02,038 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:02,067 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:02,067 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:03,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:03,157 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:03,159 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:03,159 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:03,160 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:04,068 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:04,069 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:04,283 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:04,286 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:04,286 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:04,287 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:05,069 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:05,413 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:05,415 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:05,415 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:05,416 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:06,071 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:06,071 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:06,535 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:06,537 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:06,538 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:06,539 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:07,072 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:07,662 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:07,666 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:07,666 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:07,667 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:08,073 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:08,073 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:08,667 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:08,787 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:08,789 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:08,790 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:08,791 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:09,074 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:09,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:09,913 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:09,913 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:09,914 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:10,075 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:10,076 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:11,032 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:11,035 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:11,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:11,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:11,076 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:12,077 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:12,154 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:12,156 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:12,157 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:12,158 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:13,078 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:13,370 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:13,372 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:13,373 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:13,374 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:13,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:55:13,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:55:13,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:55:14,079 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:14,079 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:14,242 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:14,491 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:14,493 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:14,493 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:14,494 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:15,080 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:15,615 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:15,617 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:15,617 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:15,618 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:16,081 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:16,081 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:16,737 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:16,740 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:16,740 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:16,741 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:17,082 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:17,858 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:17,861 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:17,861 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:17,862 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:18,083 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:18,083 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:18,980 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:18,984 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:18,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:18,985 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:19,084 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:19,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:20,085 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:20,097 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:20,100 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:20,100 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:20,101 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:21,086 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:21,222 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:21,224 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:21,224 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:21,226 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:22,086 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:22,087 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:22,342 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:22,345 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:22,345 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:22,346 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:23,087 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:23,466 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:23,469 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:23,469 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:23,471 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:24,088 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:24,088 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:24,583 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:24,586 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:24,586 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:24,587 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:25,089 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:25,588 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:25,704 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:25,730 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:25,730 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:25,732 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:26,090 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:26,090 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:26,832 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:26,834 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:26,835 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:26,836 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:27,091 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:27,955 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:27,957 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:27,957 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:27,958 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:28,092 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:28,092 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:28,966 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:55:28,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:55:28,984 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:55:28,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:55:29,178 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:29,180 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:29,180 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:29,181 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:30,094 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:30,094 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:30,287 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:30,289 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:30,290 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:30,291 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:31,095 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:31,291 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:31,401 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:31,403 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:31,403 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:31,404 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:32,096 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:32,096 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:32,523 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:32,525 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:32,526 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:32,527 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:33,097 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:33,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:33,654 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:33,654 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:33,655 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:34,099 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:34,099 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:34,785 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:34,787 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:34,787 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:34,788 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:35,100 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:35,898 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:35,900 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:35,901 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:35,902 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:36,101 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:36,101 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:36,902 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:37,032 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:37,035 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:37,035 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:37,036 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:37,102 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:38,103 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:38,175 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:38,177 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:38,177 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:38,178 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:39,104 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:39,311 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:39,313 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:39,314 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:39,315 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:40,105 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:40,105 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:40,444 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:40,447 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:40,447 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:40,448 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:41,107 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:41,579 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:41,581 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:41,581 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:41,583 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:42,108 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:42,108 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:42,583 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:42,713 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:42,715 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:42,716 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:42,717 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:43,109 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:43,845 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:43,848 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:43,848 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:43,849 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:43,984 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:55:43,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:55:43,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:55:44,110 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:44,110 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:44,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:44,986 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:44,986 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:44,987 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:45,111 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:46,112 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:46,122 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:46,124 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:46,125 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:46,126 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:47,113 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:47,349 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:47,353 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:47,353 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:47,354 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:48,114 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:48,114 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:48,354 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:48,490 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:48,493 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:48,493 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:48,495 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:49,115 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:49,627 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:49,629 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:49,630 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:49,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:50,116 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:50,116 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:50,756 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:50,758 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:50,759 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:50,760 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:51,117 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:51,877 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:51,879 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:51,879 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:51,880 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:52,118 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:52,118 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:52,992 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:52,994 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:52,995 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:52,996 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:53,119 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:53,997 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:54,111 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:54,114 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:54,114 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:54,115 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:54,120 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:54,120 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:55,239 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:55,241 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:55,242 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:55,243 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:56,121 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:56,122 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:56,367 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:56,369 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:56,370 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:56,371 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:57,122 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:57,500 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:57,503 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:57,504 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:57,505 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:58,123 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:55:58,124 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:58,679 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:58,681 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:58,682 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:58,683 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:55:58,967 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:55:58,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:55:58,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:55:58,986 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:55:59,124 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:55:59,280 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:55:59,803 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:55:59,805 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:55:59,805 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:55:59,806 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:00,126 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:00,126 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:00,924 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:00,926 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:00,926 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:00,927 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:01,126 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:02,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:02,040 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:02,041 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:02,042 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:02,128 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:02,128 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:03,159 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:03,162 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:03,162 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:03,163 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:04,129 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:04,129 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:04,281 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:04,283 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:04,284 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:04,284 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:04,285 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:05,130 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:05,506 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:05,508 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:05,508 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:05,509 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:06,131 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:06,131 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:06,622 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:06,624 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:06,624 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:06,625 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:07,132 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:07,736 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:07,738 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:07,738 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:07,739 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:08,133 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:08,133 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:08,850 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:08,854 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:08,854 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:08,855 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:09,134 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:09,855 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:09,967 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:09,969 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:09,970 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:09,971 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:10,135 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:10,135 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:11,089 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:11,092 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:11,093 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:11,094 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:11,136 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:12,137 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:12,209 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:12,211 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:12,212 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:12,213 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:13,138 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:13,325 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:13,327 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:13,327 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:13,328 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:13,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:56:13,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:56:13,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:56:14,139 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:14,139 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:14,445 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:14,446 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:14,446 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:14,447 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:15,140 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:15,487 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:15,563 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:15,566 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:15,566 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:15,567 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:16,140 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:16,141 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:16,683 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:16,685 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:16,686 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:16,687 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:17,141 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:17,801 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:17,803 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:17,804 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:17,805 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:18,142 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:18,143 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:18,920 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:18,922 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:18,923 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:18,924 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:19,143 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:20,042 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:20,045 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:20,045 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:20,046 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:20,144 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:20,144 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:21,046 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:21,168 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:21,170 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:21,170 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:21,171 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:22,146 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:22,146 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:22,381 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:22,383 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:22,384 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:22,385 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:23,147 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:23,500 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:23,502 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:23,502 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:23,503 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:24,148 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:24,148 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:24,627 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:24,630 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:24,630 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:24,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:25,149 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:25,750 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:25,753 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:25,753 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:25,754 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:26,150 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:26,150 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:26,754 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:26,877 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:26,879 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:26,880 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:26,881 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:27,151 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:27,999 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:28,001 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:28,002 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:28,003 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:28,152 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:28,152 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:28,969 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:56:28,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:56:28,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:56:28,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:56:29,116 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:29,222 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:29,223 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:29,224 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:30,154 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:30,154 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:30,234 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:30,237 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:30,237 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:30,238 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:31,155 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:31,340 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:31,342 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:31,343 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:31,344 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:32,156 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:32,157 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:32,344 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:32,443 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:32,446 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:32,446 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:32,447 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:33,158 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:33,546 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:33,548 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:33,549 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:33,550 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:34,159 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:34,159 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:34,650 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:34,652 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:34,652 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:34,653 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:35,160 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:35,754 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:35,756 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:35,757 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:35,758 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:36,161 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:36,161 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:36,857 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:36,859 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:36,859 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:36,860 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:37,163 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:37,861 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:37,960 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:37,962 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:37,962 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:37,963 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:38,164 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:38,164 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:39,154 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:39,157 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:39,157 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:39,158 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:39,165 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:40,166 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:40,275 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:40,278 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:40,278 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:40,279 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:41,167 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:41,395 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:41,397 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:41,397 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:41,398 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:42,168 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:42,168 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:42,513 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:42,515 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:42,515 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:42,516 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:43,169 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:43,517 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:43,633 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:43,635 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:43,636 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:43,637 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:43,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:56:43,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:56:43,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:56:44,170 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:44,170 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:44,757 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:44,758 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:44,759 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:44,760 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:45,171 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:45,880 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:45,882 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:45,882 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:45,883 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:46,171 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:46,172 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:46,992 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:46,994 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:46,994 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:46,995 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:47,172 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:48,097 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:48,099 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:48,100 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:48,101 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:48,173 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:48,173 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:49,101 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:49,204 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:49,208 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:49,208 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:49,209 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:50,175 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:50,175 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:50,310 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:50,312 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:50,313 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:50,314 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:51,176 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:51,428 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:51,430 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:51,430 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:51,432 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:52,177 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:52,177 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:52,553 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:52,555 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:52,556 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:52,557 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:53,178 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:53,674 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:53,676 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:53,677 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:53,678 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:54,179 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:54,179 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:54,679 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:56:54,889 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:54,891 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:54,891 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:54,893 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:55,180 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:56,012 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:56,014 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:56,014 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:56,015 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:56,181 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:56,181 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:57,132 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:57,134 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:57,134 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:57,135 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:57,182 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:58,182 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:56:58,248 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:58,250 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:58,251 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:58,252 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:56:58,969 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:56:58,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:56:58,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:56:58,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:56:59,183 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:56:59,364 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:56:59,365 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:56:59,366 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:56:59,369 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:00,184 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:00,185 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:00,408 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:00,471 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:00,473 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:00,474 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:00,475 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:01,185 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:01,583 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:01,586 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:01,586 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:01,587 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:02,186 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:02,187 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:02,710 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:02,712 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:02,713 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:02,714 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:03,187 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:03,834 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:03,837 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:03,837 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:03,838 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:04,188 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:04,189 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:04,953 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:04,955 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:04,956 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:04,957 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:05,190 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:05,957 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:06,086 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:06,088 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:06,088 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:06,089 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:06,191 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:06,191 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:07,205 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:07,208 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:07,208 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:07,209 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:08,192 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:08,192 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:08,330 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:08,333 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:08,333 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:08,334 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:09,193 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:09,452 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:09,454 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:09,455 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:09,456 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:10,194 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:10,194 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:10,582 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:10,584 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:10,585 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:10,586 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:11,195 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:11,586 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:11,801 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:11,803 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:11,804 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:11,805 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:12,197 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:12,197 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:12,928 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:12,931 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:12,931 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:12,932 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:13,198 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:13,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:57:13,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:57:13,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:57:14,053 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:14,158 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:14,158 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:14,159 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:14,199 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:14,199 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:15,173 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:15,176 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:15,176 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:15,177 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:15,199 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:16,200 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:16,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:16,295 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:16,296 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:16,297 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:17,201 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:17,297 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:17,411 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:17,413 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:17,413 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:17,415 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:18,202 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:18,202 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:18,525 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:18,528 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:18,528 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:18,529 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:19,203 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:19,649 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:19,651 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:19,651 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:19,652 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:20,204 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:20,204 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:20,765 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:20,768 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:20,768 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:20,769 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:21,205 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:21,875 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:21,877 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:21,877 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:21,879 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:22,206 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:22,206 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:22,879 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:22,988 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:22,990 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:22,991 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:22,992 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:23,207 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:24,100 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:24,125 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:24,126 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:24,127 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:24,208 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:24,208 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:25,218 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:25,220 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:25,221 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:25,222 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:25,324 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:26,209 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:26,210 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:26,611 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:26,613 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:26,614 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:26,615 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:27,210 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:27,820 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:27,823 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:27,823 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:27,825 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:28,212 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:28,212 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:28,825 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:28,940 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:28,942 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:28,943 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:28,944 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:28,969 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:57:28,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:57:28,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:57:28,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:57:29,213 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:30,058 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:30,059 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:30,060 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:30,061 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:30,214 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:30,214 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:31,176 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:31,178 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:31,179 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:31,180 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:31,215 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:32,215 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:32,289 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:32,291 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:32,292 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:32,293 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:33,216 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:33,405 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:33,407 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:33,408 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:33,409 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:34,217 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:34,217 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:34,409 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:34,527 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:34,529 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:34,530 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:34,531 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:35,218 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:35,646 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:35,648 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:35,649 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:35,650 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:36,219 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:36,219 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:36,769 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:36,771 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:36,772 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:36,773 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:37,220 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:37,881 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:37,884 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:37,884 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:37,885 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:38,221 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:38,221 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:38,998 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:39,001 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:39,001 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:39,002 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:39,222 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:40,003 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:40,113 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:40,116 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:40,116 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:40,117 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:40,223 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:40,223 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:41,234 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:41,236 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:41,236 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:41,238 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:42,224 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:42,225 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:42,350 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:42,353 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:42,353 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:42,354 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:43,225 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:43,464 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:43,467 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:43,467 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:43,468 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:44,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:57:44,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:57:44,037 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:57:44,227 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:44,227 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:44,673 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:44,674 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:44,675 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:44,676 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:45,228 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:45,715 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:45,792 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:45,795 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:45,795 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:45,796 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:46,229 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:46,229 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:46,913 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:46,915 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:46,915 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:46,916 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:47,230 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:48,031 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:48,034 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:48,034 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:48,035 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:48,230 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:48,231 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:49,144 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:49,147 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:49,147 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:49,148 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:49,231 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:50,232 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:50,255 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:50,258 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:50,258 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:50,259 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:51,233 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:51,260 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:51,366 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:51,368 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:51,369 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:51,370 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:52,234 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:52,234 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:52,480 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:52,483 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:52,483 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:52,484 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:53,235 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:53,595 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:53,597 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:53,597 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:53,598 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:54,236 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:54,236 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:54,706 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:54,708 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:54,709 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:54,710 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:55,237 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:55,821 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:55,823 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:55,824 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:55,825 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:56,238 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:56,238 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:56,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:57:56,938 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:56,940 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:56,941 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:56,942 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:57,239 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:58,048 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:58,051 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:58,051 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:58,052 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:57:58,240 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:57:58,240 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:57:58,970 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:57:59,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:57:59,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:57:59,037 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:57:59,163 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:57:59,315 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:57:59,316 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:57:59,317 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:00,241 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:00,242 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:00,282 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:00,284 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:00,285 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:00,286 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:01,243 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:01,491 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:01,493 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:01,494 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:01,495 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:02,244 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:02,244 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:02,496 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:02,610 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:02,612 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:02,613 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:02,614 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:03,244 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:03,726 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:03,728 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:03,729 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:03,730 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:04,245 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:04,246 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:04,842 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:04,844 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:04,845 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:04,846 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:05,246 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:05,955 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:05,958 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:05,958 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:05,959 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:06,247 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:06,248 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:07,071 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:07,074 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:07,074 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:07,075 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:07,248 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:08,076 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:08,186 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:08,188 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:08,189 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:08,190 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:08,249 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:08,250 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:09,308 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:09,310 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:09,310 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:09,311 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:10,251 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:10,251 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:10,417 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:10,420 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:10,420 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:10,421 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:11,252 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:11,538 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:11,540 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:11,541 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:11,542 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:12,253 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:12,253 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:12,651 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:12,653 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:12,653 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:12,654 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:13,254 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:13,655 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:13,766 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:13,769 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:13,769 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:13,770 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:14,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:58:14,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:58:14,037 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:58:14,255 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:14,256 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:14,882 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:14,884 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:14,885 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:14,886 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:15,256 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:15,996 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:15,999 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:15,999 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:16,000 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:16,257 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:16,258 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:17,109 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:17,112 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:17,112 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:17,113 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:17,258 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:18,259 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:18,323 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:18,326 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:18,326 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:18,327 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:19,260 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:19,328 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:19,440 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:19,442 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:19,443 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:19,444 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:20,261 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:20,261 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:20,558 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:20,559 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:20,560 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:20,561 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:21,262 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:21,675 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:21,678 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:21,678 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:21,679 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:22,263 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:22,263 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:22,794 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:22,796 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:22,797 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:22,798 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:23,264 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:23,916 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:23,918 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:23,919 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:23,920 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:24,265 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:24,265 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:24,920 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:25,029 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:25,031 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:25,032 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:25,033 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:25,266 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:26,152 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:26,154 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:26,154 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:26,155 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:26,267 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:26,267 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:27,268 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:27,270 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:27,270 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:27,272 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:28,269 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:28,269 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:28,385 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:28,387 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:28,387 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:28,388 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:28,971 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:58:29,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:58:29,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:58:29,078 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:58:29,270 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:29,505 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:29,507 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:29,507 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:29,508 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:30,271 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:30,271 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:30,547 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:30,623 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:30,625 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:30,626 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:30,627 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:31,272 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:31,743 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:31,745 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:31,746 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:31,747 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:32,273 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:32,273 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:32,862 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:32,864 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:32,864 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:32,865 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:33,274 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:33,983 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:33,985 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:33,985 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:33,986 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:34,274 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:34,275 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:35,195 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:35,198 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:35,198 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:35,199 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:35,276 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:36,200 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:36,276 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:36,316 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:36,318 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:36,318 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:36,319 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:37,277 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:37,430 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:37,432 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:37,432 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:37,433 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:38,278 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:38,278 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:38,548 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:38,550 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:38,550 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:38,551 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:39,279 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:39,670 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:39,672 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:39,673 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:39,674 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:40,280 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:40,280 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:40,824 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:40,826 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:40,826 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:40,827 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:41,281 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:41,828 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:41,951 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:41,954 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:41,954 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:41,955 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:42,282 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:42,282 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:43,072 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:43,074 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:43,075 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:43,076 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:43,283 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:44,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:58:44,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:58:44,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:58:44,192 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:44,283 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:44,297 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:44,298 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:44,299 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:45,284 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:45,305 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:45,307 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:45,307 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:45,308 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:46,285 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:46,286 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:46,417 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:46,419 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:46,420 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:46,421 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:47,286 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:47,421 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:47,538 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:47,541 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:47,541 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:47,542 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:48,287 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:48,288 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:48,657 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:48,659 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:48,660 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:48,661 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:49,288 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:49,769 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:49,770 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:49,771 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:49,773 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:50,289 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:50,289 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:50,886 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:50,888 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:50,889 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:50,890 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:51,290 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:52,084 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:52,087 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:52,087 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:52,088 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:52,291 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:52,291 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:53,089 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:53,196 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:53,201 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:53,202 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:53,203 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:53,292 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:54,293 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:54,311 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:54,313 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:54,314 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:54,315 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:55,294 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:55,422 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:55,425 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:55,425 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:55,426 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:56,295 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:56,295 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:56,530 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:56,532 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:56,533 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:56,534 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:57,296 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:57,646 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:57,648 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:57,649 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:57,650 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:58,297 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:58:58,297 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:58,650 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:58:58,763 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:58,765 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:58,766 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:58,767 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:58:58,972 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:58:59,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:58:59,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:58:59,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:58:59,297 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:58:59,910 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:58:59,912 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:58:59,912 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:58:59,913 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:00,298 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:00,299 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:01,027 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:01,030 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:01,030 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:01,031 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:01,299 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:02,145 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:02,148 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:02,148 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:02,149 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:02,300 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:02,301 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:03,258 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:03,261 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:03,261 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:03,262 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:03,301 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:04,263 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:04,302 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:04,371 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:04,373 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:04,373 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:04,374 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:05,303 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:05,490 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:05,493 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:05,493 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:05,494 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:06,304 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:06,304 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:06,601 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:06,604 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:06,604 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:06,605 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:07,305 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:07,797 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:07,800 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:07,800 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:07,801 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:08,306 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:08,306 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:08,913 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:08,915 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:08,916 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:08,917 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:09,307 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:09,917 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:10,031 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:10,034 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:10,034 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:10,035 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:10,308 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:10,308 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:11,147 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:11,150 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:11,150 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:11,151 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:11,309 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:12,262 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:12,264 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:12,265 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:12,266 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:12,310 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:12,310 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:13,383 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:13,385 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:13,386 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:13,387 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:14,037 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:59:14,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:59:14,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:59:14,311 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:14,311 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:14,505 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:14,507 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:14,507 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:14,508 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:15,312 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:15,548 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:15,627 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:15,629 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:15,629 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:15,631 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:16,313 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:16,313 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:16,743 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:16,746 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:16,746 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:16,747 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:17,314 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:17,858 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:17,861 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:17,862 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:17,863 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:18,315 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:18,315 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:18,979 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:18,982 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:18,982 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:18,983 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:19,316 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:20,098 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:20,124 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:20,125 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:20,127 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:20,317 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:20,317 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:21,127 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:21,218 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:21,220 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:21,221 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:21,222 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:21,318 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:22,319 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:22,332 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:22,334 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:22,334 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:22,336 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:23,320 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:23,443 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:23,445 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:23,445 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:23,447 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:24,321 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:24,321 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:24,656 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:24,658 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:24,658 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:24,660 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:25,322 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:25,765 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:25,768 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:25,768 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:25,769 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:26,323 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:26,323 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:26,769 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:26,876 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:26,878 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:26,879 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:26,880 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:27,324 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:28,000 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:28,003 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:28,004 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:28,005 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:28,325 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:28,325 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:28,973 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:59:29,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:59:29,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:59:29,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:59:29,120 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:29,273 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:29,274 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:29,275 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:29,326 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:30,243 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:30,245 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:30,246 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:30,247 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:30,327 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:30,327 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:31,360 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:31,362 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:31,362 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:31,363 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:32,328 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:32,328 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:32,364 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:32,476 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:32,478 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:32,478 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:32,479 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:33,329 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:33,593 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:33,595 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:33,596 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:33,597 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:34,330 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:34,330 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:34,717 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:34,719 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:34,719 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:34,720 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:35,331 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:35,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:35,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:35,828 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:35,830 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:36,332 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:36,332 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:36,938 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:36,940 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:36,941 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:36,942 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:37,333 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:37,942 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:38,057 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:38,059 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:38,060 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:38,061 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:38,334 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:38,334 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:39,173 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:39,175 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:39,176 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:39,177 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:39,335 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:40,335 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:40,379 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:40,381 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:40,381 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:40,382 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:41,337 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:41,491 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:41,493 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:41,494 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:41,495 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:42,338 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:42,338 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:42,608 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:42,610 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:42,611 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:42,612 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:43,339 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:43,612 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:43,725 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:43,727 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:43,728 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:43,729 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:44,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:59:44,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:59:44,039 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:59:44,340 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:44,340 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:44,852 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:44,854 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:44,854 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:44,855 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:45,341 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:45,972 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:45,974 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:45,975 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:45,976 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:46,341 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:46,342 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:47,088 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:47,090 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:47,090 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:47,092 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:47,342 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:48,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:48,209 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:48,210 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:48,211 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:48,343 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:48,344 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:49,212 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:49,328 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:49,331 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:49,331 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:49,332 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:49,344 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:50,345 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:50,450 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:50,452 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:50,453 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:50,454 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:51,346 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:51,574 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:51,576 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:51,576 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:51,577 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:52,347 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:52,347 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:52,695 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:52,697 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:52,698 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:52,699 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:53,348 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:53,813 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:53,815 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:53,816 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:53,817 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:54,349 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:54,349 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:54,817 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 17:59:54,931 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:54,933 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:54,934 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:54,935 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:55,350 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:56,049 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:56,052 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:56,052 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:56,053 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:56,351 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:56,351 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:57,162 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:57,164 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:57,164 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:57,165 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:57,352 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:58,352 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 17:59:58,373 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:58,375 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:58,376 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:58,377 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 17:59:58,974 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 17:59:59,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 17:59:59,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 17:59:59,040 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 17:59:59,353 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 17:59:59,498 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 17:59:59,500 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 17:59:59,500 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 17:59:59,501 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:00,355 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:00,355 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:00,539 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:00,616 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:00,618 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:00,618 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:00,620 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:01,356 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:01,737 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:01,739 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:01,739 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:01,741 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:02,357 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:02,357 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:02,880 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:02,882 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:02,883 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:02,884 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:03,358 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:03,991 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:03,992 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:03,993 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:03,994 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:04,359 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:04,359 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:05,107 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:05,109 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:05,109 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:05,110 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:05,360 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:06,111 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:06,242 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:06,244 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:06,245 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:06,246 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:06,360 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:06,361 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:07,353 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:07,357 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:07,357 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:07,358 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:07,361 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:08,362 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:08,469 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:08,472 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:08,472 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:08,473 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:09,363 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:09,610 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:09,613 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:09,613 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:09,614 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:10,364 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:10,364 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:10,737 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:10,740 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:10,740 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:10,741 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:11,365 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:11,742 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:11,855 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:11,857 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:11,857 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:11,858 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:12,366 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:12,367 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:12,978 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:12,980 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:12,980 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:12,981 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:13,367 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:14,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 18:00:14,039 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 18:00:14,039 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 18:00:14,104 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:14,251 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:14,251 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:14,252 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:14,368 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:14,369 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:15,333 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:15,335 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:15,336 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:15,337 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:15,370 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:16,370 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:16,466 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:16,469 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:16,469 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:16,470 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:17,371 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:17,471 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:17,598 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:17,600 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:17,601 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:17,602 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:18,372 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:18,372 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:18,712 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:18,714 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:18,715 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:18,716 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:19,373 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:19,853 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:19,856 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:19,856 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:19,857 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:20,374 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:20,374 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:20,971 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:20,974 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:20,974 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:20,975 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:21,375 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:22,091 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:22,093 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:22,093 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:22,095 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:22,376 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:22,376 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:23,095 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:23,210 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:23,213 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:23,213 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:23,214 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:23,377 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:24,358 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:24,361 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:24,361 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:24,362 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:24,378 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:24,378 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:25,469 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:25,472 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:25,473 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:25,474 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:26,380 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:26,380 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:26,579 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:26,581 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:26,582 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:26,583 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:27,381 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:27,686 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:27,689 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:27,689 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:27,690 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:28,382 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:28,382 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:28,691 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:28,841 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:28,843 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:28,844 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:28,845 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:28,974 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 18:00:29,039 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 18:00:29,039 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 18:00:29,082 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 18:00:29,383 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:29,985 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:29,987 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:29,987 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:29,988 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:30,384 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:30,384 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:31,094 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:31,096 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:31,097 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:31,098 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:31,385 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:32,293 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:32,296 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:32,296 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:32,297 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:32,386 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:32,386 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:33,410 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:33,412 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:33,413 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:33,414 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:34,387 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:34,387 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:34,414 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:34,530 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:34,532 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:34,533 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:34,534 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:35,388 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:35,655 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:35,657 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:35,658 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:35,659 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:36,389 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:36,389 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:36,768 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:36,770 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:36,770 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:36,771 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:37,390 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:37,887 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:37,889 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:37,889 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:37,891 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:38,391 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:38,392 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:38,995 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:38,997 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:38,998 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:38,999 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:39,392 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:39,999 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:40,105 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:40,107 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:40,108 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:40,109 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:40,393 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:40,394 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:41,218 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:41,221 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:41,221 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:41,223 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:41,394 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:42,336 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:42,338 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:42,338 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:42,339 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:42,395 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:42,395 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:43,451 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:43,453 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:43,453 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:43,454 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:44,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 18:00:44,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 18:00:44,039 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 18:00:44,397 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:44,397 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:44,568 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:44,570 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:44,570 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:44,571 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:45,398 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:45,611 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:45,686 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:45,688 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:45,688 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:45,689 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:46,399 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:46,399 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:46,803 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:46,805 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:46,806 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:46,807 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:47,400 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:47,918 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:47,922 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:47,922 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:47,923 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:48,401 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:48,401 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:49,137 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:49,139 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:49,140 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:49,141 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:49,402 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:50,263 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:50,265 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:50,266 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:50,267 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:50,403 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:50,403 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:51,268 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:51,379 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:51,381 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:51,381 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:51,383 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:51,404 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:52,404 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:52,483 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:52,485 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:52,485 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:52,487 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:53,405 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:53,600 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:53,603 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:53,603 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:53,604 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:54,406 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:54,407 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:54,714 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:54,716 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:54,716 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:54,717 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:55,407 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:55,826 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:55,828 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:55,829 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:55,830 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:56,408 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:56,409 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:56,830 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:00:56,947 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:56,949 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:56,950 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:56,951 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:57,409 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:58,062 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:58,064 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:58,065 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:58,066 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:58,410 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:00:58,411 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:00:58,975 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 18:00:59,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 18:00:59,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 18:00:59,039 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 18:00:59,173 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:00:59,241 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:00:59,241 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:00:59,242 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:00:59,411 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:00,298 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:00,300 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:00,300 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:00,301 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:00,412 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:00,413 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:01,432 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:01,435 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:01,435 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:01,437 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:02,414 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:02,414 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:02,437 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:01:02,548 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:02,550 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:02,551 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:02,552 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:03,415 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:03,669 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:03,671 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:03,672 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:03,673 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:04,416 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:04,417 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:04,874 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:04,877 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:04,877 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:04,878 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:05,417 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:05,995 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:05,997 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:05,998 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:05,999 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:06,419 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:06,419 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:07,113 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:07,116 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:07,116 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:07,117 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:07,420 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:08,118 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:01:08,231 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:08,235 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:08,235 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:08,236 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:08,421 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:08,421 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:09,349 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:09,351 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:09,352 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:09,353 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:09,422 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:10,422 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:10,462 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:10,465 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:10,465 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:10,466 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:10,568 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:11,423 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:11,873 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:11,876 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:11,876 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:11,877 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:12,425 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:12,425 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:12,987 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:12,990 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:12,990 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:12,991 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:13,426 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:13,991 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:01:14,038 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: stop_status +2024-08-02 18:01:14,038 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: stop_status +2024-08-02 18:01:14,040 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-02 18:01:14,104 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:14,234 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:14,235 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:14,236 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:14,427 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:14,427 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:15,217 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:15,219 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:15,220 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:15,221 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:15,428 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:16,335 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:16,338 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:16,338 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:16,339 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:16,430 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:16,430 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:17,450 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:17,452 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:17,453 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:17,454 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:18,431 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:18,431 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:18,569 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:18,572 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:18,572 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:18,573 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:19,432 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:19,574 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:01:19,682 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:19,684 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:19,684 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:19,685 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:20,433 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:20,433 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:20,797 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:20,800 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:20,800 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:20,801 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:21,434 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:22,077 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: partial_history +2024-08-02 18:01:22,102 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:22,103 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:22,104 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:22,435 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:22,435 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:23,166 DEBUG SenderThread:14040 [sender.py:send():382] send: exit +2024-08-02 18:01:23,166 INFO SenderThread:14040 [sender.py:send_exit():589] handling exit code: 255 +2024-08-02 18:01:23,167 INFO SenderThread:14040 [sender.py:send_exit():591] handling runtime: 1614 +2024-08-02 18:01:23,168 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:23,168 INFO SenderThread:14040 [sender.py:send_exit():597] send defer +2024-08-02 18:01:23,168 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,168 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-02 18:01:23,168 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,168 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-02 18:01:23,168 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 1 +2024-08-02 18:01:23,168 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,168 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-02 18:01:23,169 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,169 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-02 18:01:23,169 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 2 +2024-08-02 18:01:23,169 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,169 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-02 18:01:23,169 INFO HandlerThread:14040 [system_monitor.py:finish():203] Stopping system monitor +2024-08-02 18:01:23,169 INFO HandlerThread:14040 [interfaces.py:finish():202] Joined cpu monitor +2024-08-02 18:01:23,169 DEBUG SystemMonitor:14040 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-02 18:01:23,169 DEBUG SystemMonitor:14040 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-02 18:01:23,170 INFO HandlerThread:14040 [interfaces.py:finish():202] Joined disk monitor +2024-08-02 18:01:23,203 INFO HandlerThread:14040 [interfaces.py:finish():202] Joined gpu monitor +2024-08-02 18:01:23,203 INFO HandlerThread:14040 [interfaces.py:finish():202] Joined memory monitor +2024-08-02 18:01:23,203 INFO HandlerThread:14040 [interfaces.py:finish():202] Joined network monitor +2024-08-02 18:01:23,204 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,204 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-02 18:01:23,204 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 3 +2024-08-02 18:01:23,204 DEBUG SenderThread:14040 [sender.py:send():382] send: stats +2024-08-02 18:01:23,204 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,204 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-02 18:01:23,206 DEBUG SenderThread:14040 [sender.py:send():382] send: history +2024-08-02 18:01:23,206 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: summary_record +2024-08-02 18:01:23,207 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:23,207 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,207 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-02 18:01:23,207 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 4 +2024-08-02 18:01:23,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,207 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-02 18:01:23,207 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,207 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-02 18:01:23,207 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 5 +2024-08-02 18:01:23,207 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,208 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-02 18:01:23,208 DEBUG SenderThread:14040 [sender.py:send():382] send: summary +2024-08-02 18:01:23,209 INFO SenderThread:14040 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-02 18:01:23,209 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,209 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-02 18:01:23,209 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 6 +2024-08-02 18:01:23,209 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,209 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-02 18:01:23,210 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,210 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-02 18:01:23,210 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 7 +2024-08-02 18:01:23,210 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: status_report +2024-08-02 18:01:23,210 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:23,210 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-02 18:01:23,210 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:23,210 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-02 18:01:23,436 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json +2024-08-02 18:01:24,166 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-02 18:01:24,172 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 8 +2024-08-02 18:01:24,172 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: poll_exit +2024-08-02 18:01:24,172 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:24,173 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-02 18:01:24,173 DEBUG SenderThread:14040 [sender.py:send_request():409] send_request: defer +2024-08-02 18:01:24,173 INFO SenderThread:14040 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-02 18:01:24,173 INFO SenderThread:14040 [job_builder.py:build():296] Attempting to build job artifact +2024-08-02 18:01:24,174 INFO SenderThread:14040 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-02 18:01:24,188 INFO SenderThread:14040 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-02 18:01:24,199 INFO SenderThread:14040 [sender.py:transition_state():617] send defer: 9 +2024-08-02 18:01:24,199 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: defer +2024-08-02 18:01:24,199 DEBUG SenderThread:14040 [sender.py:send():382] send: artifact +2024-08-02 18:01:24,199 INFO HandlerThread:14040 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-02 18:01:24,437 INFO Thread-12 :14040 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240802_173428-s75vpwte/files/output.log +2024-08-02 18:01:25,167 DEBUG HandlerThread:14040 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-02 18:01:25,675 INFO wandb-upload_0:14040 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmpqzudzjxw +2024-08-02 18:01:25,787 INFO wandb-upload_1:14040 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmpj1je1_8s +2024-08-02 18:01:26,973 WARNING StreamThr :14040 [internal.py:is_dead():414] Internal process exiting, parent pid 13969 disappeared +2024-08-02 18:01:26,973 ERROR StreamThr :14040 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-02 18:01:27,047 INFO SenderThread:14040 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA4NTUyMzkyNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': None}} +2024-08-02 18:01:27,047 INFO SenderThread:14040 [sender.py:finish():1572] shutting down sender +2024-08-02 18:01:27,047 INFO SenderThread:14040 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-02 18:01:27,167 INFO HandlerThread:14040 [handler.py:finish():869] shutting down handler +2024-08-02 18:01:27,167 INFO WriterThread:14040 [datastore.py:close():296] close: /project/wandb/run-20240802_173428-s75vpwte/run-s75vpwte.wandb +2024-08-02 18:01:27,438 INFO SenderThread:14040 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240802_173428-s75vpwte/files +2024-08-02 18:01:27,438 INFO SenderThread:14040 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240802_173428-s75vpwte/files/requirements.txt requirements.txt +2024-08-02 18:01:27,438 INFO SenderThread:14040 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240802_173428-s75vpwte/files/config.yaml config.yaml +2024-08-02 18:01:27,439 INFO SenderThread:14040 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-metadata.json wandb-metadata.json +2024-08-02 18:01:27,439 INFO SenderThread:14040 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240802_173428-s75vpwte/files/wandb-summary.json wandb-summary.json +2024-08-02 18:01:27,439 INFO SenderThread:14040 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240802_173428-s75vpwte/files/output.log output.log +2024-08-02 18:01:27,439 INFO SenderThread:14040 [file_pusher.py:finish():172] shutting down file pusher +2024-08-02 18:01:27,439 INFO SenderThread:14040 [file_pusher.py:join():178] waiting for file pusher +2024-08-02 18:01:27,439 INFO SenderThread:14040 [file_stream.py:finish():595] file stream finish called +2024-08-02 18:01:28,077 INFO SenderThread:14040 [file_stream.py:finish():599] file stream finish is done diff --git a/wandb/run-20240802_173428-s75vpwte/logs/debug.log b/wandb/run-20240802_173428-s75vpwte/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..52fc8114be133b8d0c5848352b452304069ad331 --- /dev/null +++ b/wandb/run-20240802_173428-s75vpwte/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-02 17:34:28,332 INFO MainThread:13969 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Configure stats pid to 13969 +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240802_173428-s75vpwte/logs/debug.log +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240802_173428-s75vpwte/logs/debug-internal.log +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:init():566] calling init triggers +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 512, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/custom/tiny-mistral/tokenizer.model.v3', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-mistral-sample_train_2024-08-02-17:34:15', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-mistral-sample', 'save': '/work/llm_recipes/models/tiny-mistral-sample', 'base_model': '/share/pretrained_lm/custom/tiny-mistral', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-mistral-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32768, 'gradient_accumulation_steps': 40} +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:init():616] starting backend +2024-08-02 17:34:28,333 INFO MainThread:13969 [wandb_init.py:init():620] setting up manager +2024-08-02 17:34:28,339 INFO MainThread:13969 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-02 17:34:28,341 INFO MainThread:13969 [wandb_init.py:init():628] backend started and connected +2024-08-02 17:34:28,346 INFO MainThread:13969 [wandb_init.py:init():720] updated telemetry +2024-08-02 17:34:28,360 INFO MainThread:13969 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-02 17:34:28,832 INFO MainThread:13969 [wandb_run.py:_on_init():2262] communicating current version +2024-08-02 17:34:28,915 INFO MainThread:13969 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-02 17:34:28,915 INFO MainThread:13969 [wandb_init.py:init():804] starting run threads in backend +2024-08-02 17:34:28,976 INFO MainThread:13969 [wandb_run.py:_console_start():2241] atexit reg +2024-08-02 17:34:28,976 INFO MainThread:13969 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-02 17:34:28,976 INFO MainThread:13969 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-02 17:34:28,976 INFO MainThread:13969 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-02 17:34:28,977 INFO MainThread:13969 [wandb_init.py:init():847] run started, returning control to user process +2024-08-02 17:34:33,327 INFO MainThread:13969 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 256, 'model_type': 'mistral', 'max_position_embeddings': 512, 'num_attention_heads': 4, 'num_hidden_layers': 4, 'model_architecture': 'MistralForCausalLM'} +2024-08-02 17:34:33,327 INFO MainThread:13969 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml b/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b12c53cdb3d245f3a9173bc50509559917f36899 --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +valid_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +test_data_path: + desc: null + value: + - '4013541' + - /work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 256 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: Llama2Tokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: tiny-llama-sample_train_2024-08-04-13:55:35 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/tiny-llama-sample +save: + desc: null + value: /work/llm_recipes/models/tiny-llama-sample +base_model: + desc: null + value: /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 2000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 2000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 8 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 2048 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/tiny-llama-sample +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 32000 +gradient_accumulation_steps: + desc: null + value: 40 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1722747367.911791 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 2048 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 2048 +num_attention_heads: + desc: null + value: 32 +num_hidden_layers: + desc: null + value: 22 +model_architecture: + desc: null + value: LlamaForCausalLM diff --git a/wandb/run-20240804_135607-ikp7tdz1/files/output.log b/wandb/run-20240804_135607-ikp7tdz1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..392b4cab13df9a32ac703788ae02ab6233460313 --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/files/output.log @@ -0,0 +1,130 @@ +Created Hugging Face repository with ID koichi12/tiny-llama-sample. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping model loading +--> Model /share/pretrained_lm/meta-llama/TinyLlama_v1.1 +--> /share/pretrained_lm/meta-llama/TinyLlama_v1.1 has 1100.048384 Million params +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 640000 + validation: 35200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/tiny-llama-sample, skipping optimizer loading +File not found: /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/tiny-llama-sample/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(32000, 2048) + (layers): ModuleList( + (0-21): 22 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=2048, out_features=2048, bias=False) + (k_proj): Linear(in_features=2048, out_features=256, bias=False) + (v_proj): Linear(in_features=2048, out_features=256, bias=False) + (o_proj): Linear(in_features=2048, out_features=2048, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=2048, out_features=5632, bias=False) + (up_proj): Linear(in_features=2048, out_features=5632, bias=False) + (down_proj): Linear(in_features=5632, out_features=2048, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=2048, out_features=32000, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 5632, + "label_smoothing": 0.0, + "max_position_embeddings": 2048, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 32, + "num_hidden_layers": 22, + "num_key_value_heads": 4, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000.0, + "tie_word_embeddings": false, + "torch_dtype": "float32", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 32000 +} +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 281, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 110, in train + loss: torch.Tensor = model(**batch).loss + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py", line 849, in forward + output = self._fsdp_wrapped_module(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 1141, in forward + outputs = self.model( + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1511, in _wrapped_call_impl + return self._call_impl(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1520, in _call_impl + return forward_call(*args, **kwargs) + File "/project/lib/transformers/src/transformers/models/llama/modeling_llama.py", line 908, in forward + cache_position = torch.arange( +RuntimeError: CUDA error: device-side assert triggered +CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect. +For debugging consider passing CUDA_LAUNCH_BLOCKING=1. +Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions. \ No newline at end of file diff --git a/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt b/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240804_135607-ikp7tdz1/files/wandb-metadata.json b/wandb/run-20240804_135607-ikp7tdz1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..38153f3524509cd31ac916c8113552793286838d --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-04T04:56:08.637907", + "startedAt": "2024-08-04T04:56:07.879507", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "256", + "--sliding-window-size", + "2048", + "--micro-batch-size", + "8", + "--global-batch-size", + "320", + "--train-iters", + "2000", + "--tokenizer-type", + "Llama2Tokenizer", + "--tokenizer-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model", + "--train-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--valid-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--test-data-path", + "4013541", + "/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "2000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/meta-llama/TinyLlama_v1.1", + "--save", + "/work/llm_recipes/models/tiny-llama-sample", + "--load", + "/work/llm_recipes/models/tiny-llama-sample", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/tiny-llama-sample", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "tiny-llama-sample_train_2024-08-04-13:55:35" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3be5353210a678dc7008f237fa16b99f2bdf36ea" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48781967163086 + } +} diff --git a/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json b/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..619771471916cfbd0a1671799f389b4386995814 --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 67}} \ No newline at end of file diff --git a/wandb/run-20240804_135607-ikp7tdz1/logs/debug-internal.log b/wandb/run-20240804_135607-ikp7tdz1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..56d5fe6ba2121d5b9b1c463e75b194a4a7eae31c --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/logs/debug-internal.log @@ -0,0 +1,216 @@ +2024-08-04 13:56:07,912 INFO StreamThr :9151 [internal.py:wandb_internal():86] W&B internal server running at pid: 9151, started at: 2024-08-04 13:56:07.911369 +2024-08-04 13:56:07,914 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status +2024-08-04 13:56:07,916 INFO WriterThread:9151 [datastore.py:open_for_write():87] open: /project/wandb/run-20240804_135607-ikp7tdz1/run-ikp7tdz1.wandb +2024-08-04 13:56:07,917 DEBUG SenderThread:9151 [sender.py:send():382] send: header +2024-08-04 13:56:08,068 DEBUG SenderThread:9151 [sender.py:send():382] send: run +2024-08-04 13:56:08,527 INFO SenderThread:9151 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240804_135607-ikp7tdz1/files +2024-08-04 13:56:08,527 INFO SenderThread:9151 [sender.py:_start_run_threads():1136] run started: ikp7tdz1 with start time 1722747367.911791 +2024-08-04 13:56:08,532 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: check_version +2024-08-04 13:56:08,533 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: check_version +2024-08-04 13:56:08,619 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: run_start +2024-08-04 13:56:08,625 DEBUG HandlerThread:9151 [system_info.py:__init__():27] System info init +2024-08-04 13:56:08,625 DEBUG HandlerThread:9151 [system_info.py:__init__():42] System info init done +2024-08-04 13:56:08,625 INFO HandlerThread:9151 [system_monitor.py:start():194] Starting system monitor +2024-08-04 13:56:08,625 INFO SystemMonitor:9151 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-04 13:56:08,626 INFO HandlerThread:9151 [system_monitor.py:probe():214] Collecting system info +2024-08-04 13:56:08,626 INFO SystemMonitor:9151 [interfaces.py:start():190] Started cpu monitoring +2024-08-04 13:56:08,627 INFO SystemMonitor:9151 [interfaces.py:start():190] Started disk monitoring +2024-08-04 13:56:08,628 INFO SystemMonitor:9151 [interfaces.py:start():190] Started gpu monitoring +2024-08-04 13:56:08,628 INFO SystemMonitor:9151 [interfaces.py:start():190] Started memory monitoring +2024-08-04 13:56:08,629 INFO SystemMonitor:9151 [interfaces.py:start():190] Started network monitoring +2024-08-04 13:56:08,637 DEBUG HandlerThread:9151 [system_info.py:probe():151] Probing system +2024-08-04 13:56:08,639 DEBUG HandlerThread:9151 [system_info.py:_probe_git():136] Probing git +2024-08-04 13:56:08,651 DEBUG HandlerThread:9151 [system_info.py:_probe_git():144] Probing git done +2024-08-04 13:56:08,651 DEBUG HandlerThread:9151 [system_info.py:probe():199] Probing system done +2024-08-04 13:56:08,651 DEBUG HandlerThread:9151 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-04T04:56:08.637907', 'startedAt': '2024-08-04T04:56:07.879507', 'docker': None, 'cuda': None, 'args': ('--seq-length', '256', '--sliding-window-size', '2048', '--micro-batch-size', '8', '--global-batch-size', '320', '--train-iters', '2000', '--tokenizer-type', 'Llama2Tokenizer', '--tokenizer-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', '--train-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--valid-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--test-data-path', '4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '2000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', '--save', '/work/llm_recipes/models/tiny-llama-sample', '--load', '/work/llm_recipes/models/tiny-llama-sample', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/tiny-llama-sample', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'tiny-llama-sample_train_2024-08-04-13:55:35'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3be5353210a678dc7008f237fa16b99f2bdf36ea'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48781967163086}} +2024-08-04 13:56:08,651 INFO HandlerThread:9151 [system_monitor.py:probe():224] Finished collecting system info +2024-08-04 13:56:08,651 INFO HandlerThread:9151 [system_monitor.py:probe():227] Publishing system info +2024-08-04 13:56:08,653 INFO HandlerThread:9151 [system_monitor.py:probe():229] Finished publishing system info +2024-08-04 13:56:08,681 DEBUG SenderThread:9151 [sender.py:send():382] send: files +2024-08-04 13:56:08,681 INFO SenderThread:9151 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-04 13:56:08,690 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: python_packages +2024-08-04 13:56:08,690 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 13:56:08,691 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: python_packages +2024-08-04 13:56:08,691 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:56:08,692 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: stop_status +2024-08-04 13:56:08,938 DEBUG SenderThread:9151 [sender.py:send():382] send: telemetry +2024-08-04 13:56:09,405 INFO wandb-upload_0:9151 [upload_job.py:push():131] Uploaded file /tmp/tmpins_li9awandb/mkgvo0s4-wandb-metadata.json +2024-08-04 13:56:09,529 INFO Thread-12 :9151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt +2024-08-04 13:56:09,529 INFO Thread-12 :9151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_135607-ikp7tdz1/files/wandb-metadata.json +2024-08-04 13:56:10,529 INFO Thread-12 :9151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:56:12,531 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:56:13,586 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:16,533 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:56:19,567 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:23,689 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 13:56:23,690 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: stop_status +2024-08-04 13:56:23,690 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:56:24,913 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:29,913 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:34,914 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:38,689 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 13:56:38,690 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: stop_status +2024-08-04 13:56:38,732 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:56:39,955 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:40,547 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml +2024-08-04 13:56:45,164 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:50,164 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:56:53,690 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 13:56:53,690 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: stop_status +2024-08-04 13:56:53,732 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:56:55,957 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:57:00,957 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:57:05,958 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:57:08,629 DEBUG SystemMonitor:9151 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-04 13:57:08,630 DEBUG SenderThread:9151 [sender.py:send():382] send: stats +2024-08-04 13:57:08,690 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: stop_status +2024-08-04 13:57:08,690 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: stop_status +2024-08-04 13:57:08,732 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:57:11,872 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:57:15,196 DEBUG SenderThread:9151 [sender.py:send():382] send: config +2024-08-04 13:57:15,197 DEBUG SenderThread:9151 [sender.py:send():382] send: config +2024-08-04 13:57:16,571 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:57:16,600 DEBUG SenderThread:9151 [sender.py:send():382] send: exit +2024-08-04 13:57:16,601 INFO SenderThread:9151 [sender.py:send_exit():589] handling exit code: 1 +2024-08-04 13:57:16,601 INFO SenderThread:9151 [sender.py:send_exit():591] handling runtime: 67 +2024-08-04 13:57:16,602 INFO SenderThread:9151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 13:57:16,602 INFO SenderThread:9151 [sender.py:send_exit():597] send defer +2024-08-04 13:57:16,602 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,603 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-04 13:57:16,603 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,603 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-04 13:57:16,603 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 1 +2024-08-04 13:57:16,603 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,603 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-04 13:57:16,603 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,603 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-04 13:57:16,603 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 2 +2024-08-04 13:57:16,603 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,603 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-04 13:57:16,603 INFO HandlerThread:9151 [system_monitor.py:finish():203] Stopping system monitor +2024-08-04 13:57:16,603 DEBUG SystemMonitor:9151 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-04 13:57:16,604 INFO HandlerThread:9151 [interfaces.py:finish():202] Joined cpu monitor +2024-08-04 13:57:16,604 DEBUG SystemMonitor:9151 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-04 13:57:16,604 INFO HandlerThread:9151 [interfaces.py:finish():202] Joined disk monitor +2024-08-04 13:57:16,637 INFO HandlerThread:9151 [interfaces.py:finish():202] Joined gpu monitor +2024-08-04 13:57:16,637 INFO HandlerThread:9151 [interfaces.py:finish():202] Joined memory monitor +2024-08-04 13:57:16,637 INFO HandlerThread:9151 [interfaces.py:finish():202] Joined network monitor +2024-08-04 13:57:16,638 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,638 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-04 13:57:16,638 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 3 +2024-08-04 13:57:16,638 DEBUG SenderThread:9151 [sender.py:send():382] send: stats +2024-08-04 13:57:16,638 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,638 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-04 13:57:16,638 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,638 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-04 13:57:16,638 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 4 +2024-08-04 13:57:16,638 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,638 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-04 13:57:16,639 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,639 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-04 13:57:16,639 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 5 +2024-08-04 13:57:16,639 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,639 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-04 13:57:16,639 DEBUG SenderThread:9151 [sender.py:send():382] send: summary +2024-08-04 13:57:16,640 INFO SenderThread:9151 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-04 13:57:16,640 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,640 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-04 13:57:16,640 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 6 +2024-08-04 13:57:16,640 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,640 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-04 13:57:16,640 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,640 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-04 13:57:16,643 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: status_report +2024-08-04 13:57:16,835 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 7 +2024-08-04 13:57:16,836 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:16,836 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-04 13:57:16,836 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:16,836 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-04 13:57:17,572 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml +2024-08-04 13:57:17,572 INFO Thread-12 :9151 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json +2024-08-04 13:57:17,600 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:18,334 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 8 +2024-08-04 13:57:18,334 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:18,334 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:18,335 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-04 13:57:18,335 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:18,335 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-04 13:57:18,335 INFO SenderThread:9151 [job_builder.py:build():296] Attempting to build job artifact +2024-08-04 13:57:18,336 INFO SenderThread:9151 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-04 13:57:18,350 INFO SenderThread:9151 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-04 13:57:18,359 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 9 +2024-08-04 13:57:18,360 DEBUG SenderThread:9151 [sender.py:send():382] send: artifact +2024-08-04 13:57:18,360 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:18,361 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-04 13:57:18,573 INFO Thread-12 :9151 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:57:18,601 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:19,234 INFO SenderThread:9151 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTA5MTk2NTkzOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTA5MzUzODM4NQ==', 'versionIndex': 3}}} +2024-08-04 13:57:19,234 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:19,234 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-04 13:57:19,234 INFO SenderThread:9151 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-04 13:57:19,573 INFO SenderThread:9151 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240804_135607-ikp7tdz1/files +2024-08-04 13:57:19,574 INFO SenderThread:9151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt requirements.txt +2024-08-04 13:57:19,574 INFO SenderThread:9151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml config.yaml +2024-08-04 13:57:19,575 INFO SenderThread:9151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_135607-ikp7tdz1/files/wandb-metadata.json wandb-metadata.json +2024-08-04 13:57:19,576 INFO SenderThread:9151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json wandb-summary.json +2024-08-04 13:57:19,577 INFO SenderThread:9151 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log output.log +2024-08-04 13:57:19,579 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 10 +2024-08-04 13:57:19,579 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:19,579 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:19,580 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-04 13:57:19,581 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:19,581 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-04 13:57:19,581 INFO SenderThread:9151 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 13:57:19,601 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:19,601 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:19,983 INFO wandb-upload_0:9151 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_135607-ikp7tdz1/files/requirements.txt +2024-08-04 13:57:20,084 INFO wandb-upload_1:9151 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_135607-ikp7tdz1/files/config.yaml +2024-08-04 13:57:20,165 INFO wandb-upload_2:9151 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_135607-ikp7tdz1/files/wandb-summary.json +2024-08-04 13:57:20,334 INFO wandb-upload_3:9151 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240804_135607-ikp7tdz1/files/output.log +2024-08-04 13:57:20,534 INFO Thread-11 (_thread_body):9151 [sender.py:transition_state():617] send defer: 11 +2024-08-04 13:57:20,534 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:20,534 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-04 13:57:20,535 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:20,535 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-04 13:57:20,535 INFO SenderThread:9151 [file_pusher.py:join():178] waiting for file pusher +2024-08-04 13:57:20,535 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 12 +2024-08-04 13:57:20,535 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:20,535 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-04 13:57:20,535 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:20,535 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-04 13:57:20,535 INFO SenderThread:9151 [file_stream.py:finish():595] file stream finish called +2024-08-04 13:57:20,601 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:20,717 INFO SenderThread:9151 [file_stream.py:finish():599] file stream finish is done +2024-08-04 13:57:20,717 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 13 +2024-08-04 13:57:20,717 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:20,717 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:20,718 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-04 13:57:20,718 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:20,718 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-04 13:57:20,718 INFO SenderThread:9151 [sender.py:transition_state():617] send defer: 14 +2024-08-04 13:57:20,718 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: defer +2024-08-04 13:57:20,718 DEBUG SenderThread:9151 [sender.py:send():382] send: final +2024-08-04 13:57:20,718 INFO HandlerThread:9151 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-04 13:57:20,718 DEBUG SenderThread:9151 [sender.py:send():382] send: footer +2024-08-04 13:57:20,719 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: defer +2024-08-04 13:57:20,719 INFO SenderThread:9151 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-04 13:57:20,719 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:20,719 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-04 13:57:20,719 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:20,720 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: server_info +2024-08-04 13:57:20,720 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: poll_exit +2024-08-04 13:57:20,720 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: get_summary +2024-08-04 13:57:20,720 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: server_info +2024-08-04 13:57:20,721 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-04 13:57:20,722 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-04 13:57:20,722 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: job_info +2024-08-04 13:57:20,885 DEBUG SenderThread:9151 [sender.py:send_request():409] send_request: job_info +2024-08-04 13:57:20,885 INFO MainThread:9151 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-04 13:57:20,885 INFO MainThread:9151 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-04 13:57:20,885 INFO MainThread:9151 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-04 13:57:20,886 DEBUG HandlerThread:9151 [handler.py:handle_request():146] handle_request: shutdown +2024-08-04 13:57:20,886 INFO HandlerThread:9151 [handler.py:finish():869] shutting down handler +2024-08-04 13:57:21,722 INFO WriterThread:9151 [datastore.py:close():296] close: /project/wandb/run-20240804_135607-ikp7tdz1/run-ikp7tdz1.wandb +2024-08-04 13:57:21,885 INFO SenderThread:9151 [sender.py:finish():1572] shutting down sender +2024-08-04 13:57:21,885 INFO SenderThread:9151 [file_pusher.py:finish():172] shutting down file pusher +2024-08-04 13:57:21,885 INFO SenderThread:9151 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240804_135607-ikp7tdz1/logs/debug.log b/wandb/run-20240804_135607-ikp7tdz1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3ab19444693bce4b2e670c9afbee5a94c7fe326c --- /dev/null +++ b/wandb/run-20240804_135607-ikp7tdz1/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Configure stats pid to 9079 +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train tuny llama sample'} +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240804_135607-ikp7tdz1/logs/debug.log +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240804_135607-ikp7tdz1/logs/debug-internal.log +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_init.py:init():566] calling init triggers +2024-08-04 13:56:07,904 INFO MainThread:9079 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'valid_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'test_data_path': ['4013541', '/work/llm_recipes/datasets/bin/common_crawl_and_extended_common_crawl.doc_extracted.200.sorted.uniq.filtered.shuf.head/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 256, 'num_workers': 2, 'tokenizer_type': 'Llama2Tokenizer', 'tokenizer_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1/tokenizer.model', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'tiny-llama-sample_train_2024-08-04-13:55:35', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/tiny-llama-sample', 'save': '/work/llm_recipes/models/tiny-llama-sample', 'base_model': '/share/pretrained_lm/meta-llama/TinyLlama_v1.1', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 2000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 2000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 8, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 2048, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/tiny-llama-sample', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 32000, 'gradient_accumulation_steps': 40} +2024-08-04 13:56:07,905 INFO MainThread:9079 [wandb_init.py:init():616] starting backend +2024-08-04 13:56:07,905 INFO MainThread:9079 [wandb_init.py:init():620] setting up manager +2024-08-04 13:56:07,909 INFO MainThread:9079 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-04 13:56:07,911 INFO MainThread:9079 [wandb_init.py:init():628] backend started and connected +2024-08-04 13:56:07,916 INFO MainThread:9079 [wandb_init.py:init():720] updated telemetry +2024-08-04 13:56:08,064 INFO MainThread:9079 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-04 13:56:08,532 INFO MainThread:9079 [wandb_run.py:_on_init():2262] communicating current version +2024-08-04 13:56:08,612 INFO MainThread:9079 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-04 13:56:08,612 INFO MainThread:9079 [wandb_init.py:init():804] starting run threads in backend +2024-08-04 13:56:08,689 INFO MainThread:9079 [wandb_run.py:_console_start():2241] atexit reg +2024-08-04 13:56:08,689 INFO MainThread:9079 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-04 13:56:08,690 INFO MainThread:9079 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-04 13:56:08,690 INFO MainThread:9079 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-04 13:56:08,691 INFO MainThread:9079 [wandb_init.py:init():847] run started, returning control to user process +2024-08-04 13:57:15,195 INFO MainThread:9079 [wandb_run.py:_config_callback():1343] config_cb None None {'activation_function': 'silu', 'hidden_size': 2048, 'model_type': 'llama', 'max_position_embeddings': 2048, 'num_attention_heads': 32, 'num_hidden_layers': 22, 'model_architecture': 'LlamaForCausalLM'} +2024-08-04 13:57:15,196 INFO MainThread:9079 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-08-04 13:57:21,887 WARNING MsgRouterThr:9079 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240804_135607-ikp7tdz1/run-ikp7tdz1.wandb b/wandb/run-20240804_135607-ikp7tdz1/run-ikp7tdz1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..1816bb7d3ffde368f297cfd7c1f2af4d840f0925 Binary files /dev/null and b/wandb/run-20240804_135607-ikp7tdz1/run-ikp7tdz1.wandb differ diff --git a/wandb/run-20240812_070449-ufge4h1y/files/config.yaml b/wandb/run-20240812_070449-ufge4h1y/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf052ef2a43c864792bdfcca2ae7a317f85af172 --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-qwen2-0.5B_train_2024-08-12-07:04:37 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 5 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723413889.11596 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240812_070449-ufge4h1y/files/output.log b/wandb/run-20240812_070449-ufge4h1y/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..478b15e96333175f4651c5b6ed6bc1ba8440e9e8 --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/files/output.log @@ -0,0 +1,158 @@ +Created Hugging Face repository with ID koichi12/yans-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-qwen2-0.5B, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 323200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-qwen2-0.5B, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 69.43623917184445, Tokens per sec: 17268.44384112612, Loss: 4.1814446449279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 69.64205785663373, Tokens per sec: 17319.629914020166, Loss: 4.191491603851318 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 69.60094665048808, Tokens per sec: 17309.405763590446, Loss: 4.197597026824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 69.47512522949748, Tokens per sec: 17278.114608304662, Loss: 4.183670520782471 +------------------------------------------------------------------ +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 69.67467547447801, Tokens per sec: 17327.7417517103, Loss: 4.198245048522949 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/model.pt +[rank0]:[2024-08-12 07:11:16,345] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006517466999866883, 'preprocessing_with_comm': 0.0007555539996246807, 'state_converting': 0.9849483990001318, : 0.9936859660001574}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000005, took 4.44s +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 70.22008480550622, Tokens per sec: 17463.382312253587, Loss: 4.179391860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 69.98955682269778, Tokens per sec: 17406.051161079293, Loss: 4.190949440002441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 69.94509258955091, Tokens per sec: 17394.993129679646, Loss: 4.189082622528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 70.07602036768274, Tokens per sec: 17427.55421033261, Loss: 4.181089878082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 70.03395601975187, Tokens per sec: 17417.093018329397, Loss: 4.1603803634643555 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/model.pt +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/model.pt +[rank0]:[2024-08-12 07:17:37,283] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0064329239994549425, 'preprocessing_with_comm': 0.0007190309997895383, 'state_converting': 0.9757228209991808, : 0.9842789310005173}) +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000010, took 4.48s +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 70.31766010694388, Tokens per sec: 17487.64879951231, Loss: 4.118324279785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 70.37958976318761, Tokens per sec: 17503.050393891557, Loss: 4.171144008636475 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240812_070449-ufge4h1y/files/requirements.txt b/wandb/run-20240812_070449-ufge4h1y/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240812_070449-ufge4h1y/files/wandb-metadata.json b/wandb/run-20240812_070449-ufge4h1y/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c87bf87a20a4afb992a9a6996244a0f45f60a34a --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-11T22:04:49.754332", + "startedAt": "2024-08-11T22:04:49.102690", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--test-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "5", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-qwen2-0.5B_train_2024-08-12-07:04:37" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0429999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487823486328125 + } +} diff --git a/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json b/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..0268249445b4471dac16302bf1ee648d7248775d --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 4.171144008636475, "training/perplexity": 64.78952950804121, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 12, "optimizer/lr": 1.4560000000000001e-06, "optimizer/variance_l2": 0.012989128226478895, "optimizer/variance_sqrt_l2": 0.6784465027663834, "optimizer/momentum_l2": 0.7107880089338467, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.4604034423828125, "optimizer/variance_sqrt_l1": 2849.0, "optimizer/momentum_l1": 2785.25, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.00909423828125, "optimizer/variance_sqrt_abs_max": 0.09521484375, "optimizer/momentum_abs_max": 0.10107421875, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 74.9035151299995, "stats/tokens_per_sec": 17503.050393891557, "stats/tokens_per_sec_per_gpu": 17503.050393891557, "stats/tflops": 70.37958976318761, "_timestamp": 1723414808.909133, "_runtime": 919.7931730747223, "_step": 12, "_wandb": {"runtime": 922}} \ No newline at end of file diff --git a/wandb/run-20240812_070449-ufge4h1y/logs/debug-internal.log b/wandb/run-20240812_070449-ufge4h1y/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..897325fd59f3d1e6288c619810903047e46512ec --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/logs/debug-internal.log @@ -0,0 +1,616 @@ +2024-08-12 07:04:49,117 INFO StreamThr :13762 [internal.py:wandb_internal():86] W&B internal server running at pid: 13762, started at: 2024-08-12 07:04:49.116639 +2024-08-12 07:04:49,119 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status +2024-08-12 07:04:49,121 INFO WriterThread:13762 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_070449-ufge4h1y/run-ufge4h1y.wandb +2024-08-12 07:04:49,122 DEBUG SenderThread:13762 [sender.py:send():382] send: header +2024-08-12 07:04:49,136 DEBUG SenderThread:13762 [sender.py:send():382] send: run +2024-08-12 07:04:49,638 INFO SenderThread:13762 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_070449-ufge4h1y/files +2024-08-12 07:04:49,638 INFO SenderThread:13762 [sender.py:_start_run_threads():1136] run started: ufge4h1y with start time 1723413889.11596 +2024-08-12 07:04:49,643 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: check_version +2024-08-12 07:04:49,643 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: check_version +2024-08-12 07:04:49,733 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: run_start +2024-08-12 07:04:49,739 DEBUG HandlerThread:13762 [system_info.py:__init__():27] System info init +2024-08-12 07:04:49,739 DEBUG HandlerThread:13762 [system_info.py:__init__():42] System info init done +2024-08-12 07:04:49,740 INFO HandlerThread:13762 [system_monitor.py:start():194] Starting system monitor +2024-08-12 07:04:49,740 INFO SystemMonitor:13762 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-12 07:04:49,740 INFO HandlerThread:13762 [system_monitor.py:probe():214] Collecting system info +2024-08-12 07:04:49,740 INFO SystemMonitor:13762 [interfaces.py:start():190] Started cpu monitoring +2024-08-12 07:04:49,741 INFO SystemMonitor:13762 [interfaces.py:start():190] Started disk monitoring +2024-08-12 07:04:49,741 INFO SystemMonitor:13762 [interfaces.py:start():190] Started gpu monitoring +2024-08-12 07:04:49,742 INFO SystemMonitor:13762 [interfaces.py:start():190] Started memory monitoring +2024-08-12 07:04:49,744 INFO SystemMonitor:13762 [interfaces.py:start():190] Started network monitoring +2024-08-12 07:04:49,754 DEBUG HandlerThread:13762 [system_info.py:probe():151] Probing system +2024-08-12 07:04:49,756 DEBUG HandlerThread:13762 [system_info.py:_probe_git():136] Probing git +2024-08-12 07:04:49,770 DEBUG HandlerThread:13762 [system_info.py:_probe_git():144] Probing git done +2024-08-12 07:04:49,771 DEBUG HandlerThread:13762 [system_info.py:probe():199] Probing system done +2024-08-12 07:04:49,771 DEBUG HandlerThread:13762 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T22:04:49.754332', 'startedAt': '2024-08-11T22:04:49.102690', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--valid-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--test-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '5', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-qwen2-0.5B', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-qwen2-0.5B_train_2024-08-12-07:04:37'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}} +2024-08-12 07:04:49,771 INFO HandlerThread:13762 [system_monitor.py:probe():224] Finished collecting system info +2024-08-12 07:04:49,771 INFO HandlerThread:13762 [system_monitor.py:probe():227] Publishing system info +2024-08-12 07:04:49,772 INFO HandlerThread:13762 [system_monitor.py:probe():229] Finished publishing system info +2024-08-12 07:04:49,779 DEBUG SenderThread:13762 [sender.py:send():382] send: files +2024-08-12 07:04:49,779 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-12 07:04:49,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: python_packages +2024-08-12 07:04:49,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:04:49,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:04:49,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: python_packages +2024-08-12 07:04:49,791 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:04:50,088 DEBUG SenderThread:13762 [sender.py:send():382] send: telemetry +2024-08-12 07:04:50,465 INFO wandb-upload_0:13762 [upload_job.py:push():131] Uploaded file /tmp/tmp0h3j51sdwandb/z7nk28zc-wandb-metadata.json +2024-08-12 07:04:50,640 INFO Thread-12 :13762 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-metadata.json +2024-08-12 07:04:50,640 INFO Thread-12 :13762 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:04:50,640 INFO Thread-12 :13762 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_070449-ufge4h1y/files/requirements.txt +2024-08-12 07:04:52,640 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:04:54,468 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:04:54,641 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:04:54,719 DEBUG SenderThread:13762 [sender.py:send():382] send: config +2024-08-12 07:04:54,719 DEBUG SenderThread:13762 [sender.py:send():382] send: config +2024-08-12 07:04:56,643 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:04:59,720 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:04,721 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:04,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:05:04,790 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:05:04,790 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:05:10,015 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:15,015 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:19,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:05:19,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:05:19,828 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:05:20,046 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:20,658 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/config.yaml +2024-08-12 07:05:25,253 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:30,254 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:34,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:05:34,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:05:34,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:05:36,061 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:41,062 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:46,063 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:49,744 DEBUG SystemMonitor:13762 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-12 07:05:49,746 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:05:49,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:05:49,788 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:05:49,828 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:05:51,986 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:05:56,987 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:01,988 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:04,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:06:04,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:06:04,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:06:06,993 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:10,837 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:06:12,691 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:06:12,882 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:17,882 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:19,747 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:06:19,788 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:06:19,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:06:19,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:06:23,039 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:28,039 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:33,040 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:34,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:06:34,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:06:34,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:06:39,036 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:44,037 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:49,037 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:49,748 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:06:49,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:06:49,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:06:49,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:06:54,988 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:06:59,989 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:04,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:07:04,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:07:04,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:07:05,036 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:10,037 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:15,038 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:19,749 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:07:19,789 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:07:19,789 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:07:19,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:07:20,985 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:25,986 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:26,535 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:07:26,538 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:07:26,538 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:07:26,540 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:07:26,739 INFO Thread-12 :13762 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:07:28,741 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:07:31,578 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:34,791 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:07:34,791 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:07:34,791 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:07:37,002 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:42,003 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:47,004 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:49,750 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:07:49,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:07:49,792 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:07:49,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:07:52,985 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:07:57,986 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:02,986 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:04,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:08:04,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:08:04,793 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:08:08,037 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:13,038 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:18,039 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:19,751 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:08:19,791 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:08:19,792 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:08:19,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:08:23,989 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:28,990 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:33,991 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:34,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:08:34,792 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:08:34,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:08:39,042 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:42,279 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:08:42,281 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:08:42,282 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:08:42,283 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:08:42,792 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:08:44,322 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:44,793 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:08:49,322 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:49,752 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:08:49,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:08:49,792 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:08:49,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:08:54,999 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:08:59,999 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:04,792 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:09:04,793 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:09:04,832 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:09:05,001 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:10,002 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:15,003 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:19,753 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:09:19,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:09:19,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:09:19,793 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:09:20,044 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:25,045 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:30,046 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:34,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:09:34,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:09:34,793 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:09:35,995 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:40,995 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:45,996 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:49,754 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:09:49,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:09:49,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:09:49,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:09:51,979 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:56,980 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:09:58,160 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:09:58,162 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:09:58,162 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:09:58,163 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:09:58,845 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:10:00,846 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:10:02,202 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:04,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:10:04,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:10:04,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:10:08,061 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:13,062 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:18,063 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:19,755 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:10:19,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:10:19,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:10:19,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:10:23,070 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:28,071 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:33,072 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:34,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:10:34,795 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:10:34,795 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:10:38,976 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:43,977 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:48,978 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:49,758 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:10:49,793 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:10:49,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:10:49,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:10:54,010 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:10:59,011 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:04,012 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:04,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:11:04,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:11:04,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:11:09,041 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:13,824 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:11:13,826 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:11:13,826 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:11:13,827 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:11:13,896 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:11:14,866 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:14,897 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:11:16,898 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:11:18,900 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:11:19,757 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:11:19,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:11:19,794 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:11:19,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:11:20,004 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:20,901 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:11:25,004 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:30,005 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:34,794 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:11:34,795 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:11:34,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:11:35,993 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:40,994 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:45,994 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:49,758 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:11:49,795 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:11:49,795 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:11:49,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:11:51,989 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:11:56,990 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:01,990 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:04,795 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:12:04,795 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:12:04,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:12:06,998 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:11,999 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:17,000 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:19,760 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:12:19,795 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:12:19,795 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:12:19,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:12:22,010 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:27,011 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:32,011 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:33,344 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:12:33,346 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:12:33,346 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:12:33,348 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:12:33,948 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:12:34,796 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:12:34,796 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:12:34,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:12:34,948 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:12:38,002 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:43,002 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:48,003 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:49,760 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:12:49,795 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:12:49,796 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:12:49,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:12:53,056 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:12:58,057 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:03,057 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:04,796 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:13:04,796 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:13:04,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:13:09,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:14,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:19,034 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:19,761 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:13:19,796 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:13:19,796 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:13:19,836 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:13:25,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:30,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:34,838 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:13:34,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:13:34,839 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:13:35,055 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:40,056 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:45,057 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:48,668 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:13:48,670 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:13:48,671 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:13:48,672 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:13:48,997 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:13:49,762 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:13:49,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:13:49,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:13:49,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:13:50,998 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:13:51,022 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:13:56,023 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:01,023 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:04,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:14:04,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:14:04,839 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:14:06,089 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:11,090 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:16,090 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:19,763 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:14:19,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:14:19,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:14:19,839 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:14:21,108 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:26,109 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:31,109 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:34,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:14:34,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:14:34,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:14:37,031 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:42,032 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:47,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:49,764 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:14:49,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:14:49,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:14:49,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:14:52,060 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:14:57,061 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:02,061 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:04,039 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:15:04,041 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:15:04,041 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:15:04,043 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:15:04,047 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:15:04,839 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:15:04,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:15:04,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:15:05,048 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:15:07,077 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:12,077 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:17,078 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:19,765 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:15:19,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:15:19,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:15:19,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:15:22,080 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:27,081 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:32,082 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:34,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:15:34,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:15:34,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:15:38,041 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:43,042 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:48,042 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:49,766 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:15:49,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:15:49,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:15:49,840 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:15:53,080 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:15:58,080 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:03,081 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:04,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:16:04,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:16:04,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:16:09,051 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:14,052 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:19,053 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:19,269 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:16:19,271 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:16:19,271 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:16:19,273 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:16:19,767 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:16:19,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:16:19,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:16:19,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:16:20,099 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:16:21,099 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:16:25,052 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:30,052 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:34,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:16:34,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:16:34,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:16:35,100 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:40,100 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:45,101 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:49,768 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:16:49,840 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:16:49,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:16:49,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:16:51,038 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:16:56,039 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:01,040 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:04,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:17:04,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:17:04,841 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:17:06,122 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:11,123 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:16,124 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:19,769 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:17:19,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:17:19,841 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:17:19,842 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:17:22,020 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:27,021 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:32,022 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:34,545 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:17:34,548 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:17:34,548 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:17:34,550 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:17:35,013 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:17:35,041 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:17:35,041 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:17:35,149 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:17:37,151 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:17:37,272 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:39,152 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:17:41,154 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:17:43,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:48,033 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:49,770 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:17:49,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:17:49,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:17:49,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:17:53,197 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:17:58,198 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:03,198 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:04,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:18:04,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:18:04,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:18:08,232 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:13,233 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:18,233 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:19,771 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:18:19,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:18:19,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:18:19,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:18:23,237 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:28,237 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:33,238 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:34,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:18:34,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:18:34,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:18:39,167 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:44,168 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:49,168 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:49,772 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:18:49,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:18:49,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:18:49,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:18:54,004 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:18:54,006 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:18:54,007 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:18:54,008 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:18:54,198 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:18:55,009 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:18:55,199 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:19:00,010 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:04,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:19:04,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:19:04,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:19:05,244 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:10,245 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:15,245 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:19,773 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:19:19,970 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:19:19,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:19:19,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:19:21,167 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:26,168 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:31,169 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:34,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:19:34,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:19:34,972 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:19:37,149 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:42,150 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:47,151 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:49,774 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:19:49,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:19:49,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:19:49,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:19:52,230 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:19:57,230 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:02,231 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:04,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:20:04,971 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:20:04,971 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:20:08,210 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:08,910 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:20:08,913 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:20:08,913 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:20:08,914 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:20:09,243 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:20:09,244 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:20:12,332 DEBUG SenderThread:13762 [sender.py:send():382] send: exit +2024-08-12 07:20:12,332 INFO SenderThread:13762 [sender.py:send_exit():589] handling exit code: 255 +2024-08-12 07:20:12,332 INFO SenderThread:13762 [sender.py:send_exit():591] handling runtime: 922 +2024-08-12 07:20:12,333 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:20:12,334 INFO SenderThread:13762 [sender.py:send_exit():597] send defer +2024-08-12 07:20:12,334 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,334 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-12 07:20:12,334 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,334 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-12 07:20:12,334 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 1 +2024-08-12 07:20:12,334 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,334 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-12 07:20:12,334 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,334 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-12 07:20:12,334 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 2 +2024-08-12 07:20:12,335 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,335 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-12 07:20:12,335 INFO HandlerThread:13762 [system_monitor.py:finish():203] Stopping system monitor +2024-08-12 07:20:12,335 DEBUG SystemMonitor:13762 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-12 07:20:12,335 INFO HandlerThread:13762 [interfaces.py:finish():202] Joined cpu monitor +2024-08-12 07:20:12,335 DEBUG SystemMonitor:13762 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-12 07:20:12,335 INFO HandlerThread:13762 [interfaces.py:finish():202] Joined disk monitor +2024-08-12 07:20:12,371 INFO HandlerThread:13762 [interfaces.py:finish():202] Joined gpu monitor +2024-08-12 07:20:12,371 INFO HandlerThread:13762 [interfaces.py:finish():202] Joined memory monitor +2024-08-12 07:20:12,371 INFO HandlerThread:13762 [interfaces.py:finish():202] Joined network monitor +2024-08-12 07:20:12,372 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,372 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-12 07:20:12,372 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 3 +2024-08-12 07:20:12,372 DEBUG SenderThread:13762 [sender.py:send():382] send: stats +2024-08-12 07:20:12,372 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,373 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-12 07:20:12,374 DEBUG SenderThread:13762 [sender.py:send():382] send: history +2024-08-12 07:20:12,374 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:20:12,375 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:20:12,376 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,376 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-12 07:20:12,376 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 4 +2024-08-12 07:20:12,376 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,376 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-12 07:20:12,376 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,376 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-12 07:20:12,376 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 5 +2024-08-12 07:20:12,376 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,376 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-12 07:20:12,377 DEBUG SenderThread:13762 [sender.py:send():382] send: summary +2024-08-12 07:20:12,378 INFO SenderThread:13762 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:20:12,378 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,378 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-12 07:20:12,378 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 6 +2024-08-12 07:20:12,378 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,378 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-12 07:20:12,379 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,379 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-12 07:20:12,379 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 7 +2024-08-12 07:20:12,379 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:12,379 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:12,379 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-12 07:20:12,379 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:12,379 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-12 07:20:13,247 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json +2024-08-12 07:20:13,332 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 07:20:15,017 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 8 +2024-08-12 07:20:15,017 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 07:20:15,017 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:15,017 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-12 07:20:15,018 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:15,018 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-12 07:20:15,018 INFO SenderThread:13762 [job_builder.py:build():296] Attempting to build job artifact +2024-08-12 07:20:15,019 INFO SenderThread:13762 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-12 07:20:15,033 INFO SenderThread:13762 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-12 07:20:15,042 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 9 +2024-08-12 07:20:15,042 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:15,042 DEBUG SenderThread:13762 [sender.py:send():382] send: artifact +2024-08-12 07:20:15,042 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-12 07:20:15,248 INFO Thread-12 :13762 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log +2024-08-12 07:20:15,333 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 07:20:15,953 INFO SenderThread:13762 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTEzOTg5OTc5MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-12 07:20:15,953 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:15,953 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-12 07:20:15,953 INFO SenderThread:13762 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-12 07:20:16,249 INFO SenderThread:13762 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_070449-ufge4h1y/files +2024-08-12 07:20:16,249 INFO SenderThread:13762 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_070449-ufge4h1y/files/requirements.txt requirements.txt +2024-08-12 07:20:16,250 INFO SenderThread:13762 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_070449-ufge4h1y/files/config.yaml config.yaml +2024-08-12 07:20:16,250 INFO SenderThread:13762 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-metadata.json wandb-metadata.json +2024-08-12 07:20:16,250 INFO SenderThread:13762 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_070449-ufge4h1y/files/wandb-summary.json wandb-summary.json +2024-08-12 07:20:16,250 INFO SenderThread:13762 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_070449-ufge4h1y/files/output.log output.log +2024-08-12 07:20:16,250 INFO SenderThread:13762 [sender.py:transition_state():617] send defer: 10 +2024-08-12 07:20:16,250 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 07:20:16,251 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:20:16,251 INFO HandlerThread:13762 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-12 07:20:16,251 DEBUG SenderThread:13762 [sender.py:send_request():409] send_request: defer +2024-08-12 07:20:16,251 INFO SenderThread:13762 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-12 07:20:16,251 INFO SenderThread:13762 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 07:20:20,252 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:25,252 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:30,253 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:35,254 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:40,254 DEBUG HandlerThread:13762 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:20:43,105 WARNING StreamThr :13762 [internal.py:is_dead():414] Internal process exiting, parent pid 13691 disappeared +2024-08-12 07:20:43,105 ERROR StreamThr :13762 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-12 07:20:43,255 INFO SenderThread:13762 [sender.py:finish():1572] shutting down sender +2024-08-12 07:20:43,255 INFO SenderThread:13762 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 07:20:43,255 INFO HandlerThread:13762 [handler.py:finish():869] shutting down handler +2024-08-12 07:20:43,255 INFO SenderThread:13762 [file_pusher.py:join():178] waiting for file pusher +2024-08-12 07:20:43,255 INFO WriterThread:13762 [datastore.py:close():296] close: /project/wandb/run-20240812_070449-ufge4h1y/run-ufge4h1y.wandb +2024-08-12 07:20:43,255 INFO SenderThread:13762 [file_stream.py:finish():595] file stream finish called +2024-08-12 07:20:43,425 INFO SenderThread:13762 [file_stream.py:finish():599] file stream finish is done diff --git a/wandb/run-20240812_070449-ufge4h1y/logs/debug.log b/wandb/run-20240812_070449-ufge4h1y/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b7e967529d7f1a094b496400784615817ccbba30 --- /dev/null +++ b/wandb/run-20240812_070449-ufge4h1y/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-12 07:04:49,108 INFO MainThread:13691 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Configure stats pid to 13691 +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'} +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_070449-ufge4h1y/logs/debug.log +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_070449-ufge4h1y/logs/debug-internal.log +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:init():566] calling init triggers +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-12-07:04:37', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 5, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320} +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:init():616] starting backend +2024-08-12 07:04:49,109 INFO MainThread:13691 [wandb_init.py:init():620] setting up manager +2024-08-12 07:04:49,114 INFO MainThread:13691 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-12 07:04:49,115 INFO MainThread:13691 [wandb_init.py:init():628] backend started and connected +2024-08-12 07:04:49,120 INFO MainThread:13691 [wandb_init.py:init():720] updated telemetry +2024-08-12 07:04:49,131 INFO MainThread:13691 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-12 07:04:49,642 INFO MainThread:13691 [wandb_run.py:_on_init():2262] communicating current version +2024-08-12 07:04:49,725 INFO MainThread:13691 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-12 07:04:49,725 INFO MainThread:13691 [wandb_init.py:init():804] starting run threads in backend +2024-08-12 07:04:49,788 INFO MainThread:13691 [wandb_run.py:_console_start():2241] atexit reg +2024-08-12 07:04:49,788 INFO MainThread:13691 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-12 07:04:49,788 INFO MainThread:13691 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-12 07:04:49,788 INFO MainThread:13691 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-12 07:04:49,789 INFO MainThread:13691 [wandb_init.py:init():847] run started, returning control to user process +2024-08-12 07:04:54,718 INFO MainThread:13691 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-12 07:04:54,718 INFO MainThread:13691 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240812_070449-ufge4h1y/run-ufge4h1y.wandb b/wandb/run-20240812_070449-ufge4h1y/run-ufge4h1y.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e2c79872dd2a88ac462dd60746d83be55a2bf3ae Binary files /dev/null and b/wandb/run-20240812_070449-ufge4h1y/run-ufge4h1y.wandb differ diff --git a/wandb/run-20240812_073202-yby212na/files/config.yaml b/wandb/run-20240812_073202-yby212na/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b2262a41f149c7562ebe1ce73fcecf7b3ea9fe0 --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/files/config.yaml @@ -0,0 +1,335 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-qwen2-0.5B_train_2024-08-12-07:31:51 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 5 +save_interval: + desc: null + value: 1 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723415522.366221 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240812_073202-yby212na/files/output.log b/wandb/run-20240812_073202-yby212na/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..0392d9066369f95904f87035741424a3000e7be7 --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/files/output.log @@ -0,0 +1,116 @@ +Created Hugging Face repository with ID koichi12/yans-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-qwen2-0.5B, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 12803200 + test: 3200 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-qwen2-0.5B, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-qwen2-0.5B/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 69.93553660778689, Tokens per sec: 17392.616605023257, Loss: 4.1814446449279785 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001 +Saving model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/optimizer.pt +[rank0]:[2024-08-12 07:33:22,462] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.006542664999869885, 'preprocessing_with_comm': 0.0007797380003466969, 'state_converting': 0.9963913259998662, : 1.0051406040001893}) +Saved optimizer state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001/rng.pt +None +/work/llm_recipes/models/yans-qwen2-0.5B/tokenizer +Saved checkpoint to /work/llm_recipes/models/yans-qwen2-0.5B/iter_0000001, took 4.39s \ No newline at end of file diff --git a/wandb/run-20240812_073202-yby212na/files/requirements.txt b/wandb/run-20240812_073202-yby212na/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7c90fd1c54d0a9881f6b4c6465b2a4fa88c9056c --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/files/requirements.txt @@ -0,0 +1,271 @@ +absl-py==2.1.0 +accelerate==0.33.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +distributed==2023.11.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +grpcio==1.60.1 +huggingface-hub==0.24.5 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +jedi==0.19.1 +jinja2==3.1.3 +joblib==1.3.2 +json5==0.9.14 +jsonnet==0.19.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langcodes==3.3.0 +lazy-loader==0.3 +librosa==0.10.1 +llvmlite==0.40.1 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +murmurhash==1.0.10 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +onnx==1.15.0rc2 +opencv==4.7.0 +optree==0.10.0 +packaging==23.2 +pandas==1.5.3 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +peft==0.11.1 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.1.dev0+gba5374836.d20240125 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pynvml==11.4.1 +pyparsing==3.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.31.0 +rich==13.7.0 +rmm==23.12.0 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.0 +safetensors==0.4.3 +scikit-learn==1.2.0 +scipy==1.12.0 +send2trash==1.8.2 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +srsly==2.4.8 +stack-data==0.6.3 +sympy==1.12 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm==4.66.1 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.9.0 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +xdoctest==1.0.2 +xgboost==1.7.6 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 \ No newline at end of file diff --git a/wandb/run-20240812_073202-yby212na/files/wandb-metadata.json b/wandb/run-20240812_073202-yby212na/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bea34fdb458e7c6b29b59101cc6a49c2bfef8015 --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-11T22:32:03.032279", + "startedAt": "2024-08-11T22:32:02.353340", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--test-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "1", + "--eval-interval", + "5", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-qwen2-0.5B_train_2024-08-12-07:31:51" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0429999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.043, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.487823486328125 + } +} diff --git a/wandb/run-20240812_073202-yby212na/files/wandb-summary.json b/wandb/run-20240812_073202-yby212na/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9c977c2510290d009c62124ead3523fb4f4139df --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 132}, "training/loss": 4.1814446449279785, "training/perplexity": 65.46035190441053, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 1, "optimizer/lr": 1.038e-06, "optimizer/variance_l2": 0.001437161465185535, "optimizer/variance_sqrt_l2": 0.22307888709863474, "optimizer/momentum_l2": 0.09989735636562776, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.04984140396118164, "optimizer/variance_sqrt_l1": 889.25, "optimizer/momentum_l1": 397.875, "optimizer/weight_l1": 6886400.0, "optimizer/variance_abs_max": 0.00101470947265625, "optimizer/variance_sqrt_abs_max": 0.03173828125, "optimizer/momentum_abs_max": 0.0142822265625, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 75.37911228499979, "stats/tokens_per_sec": 17392.616605023257, "stats/tokens_per_sec_per_gpu": 17392.616605023257, "stats/tflops": 69.93553660778689, "_timestamp": 1723415599.9530108, "_runtime": 77.58678984642029, "_step": 1} \ No newline at end of file diff --git a/wandb/run-20240812_073202-yby212na/logs/debug-internal.log b/wandb/run-20240812_073202-yby212na/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c8584877225f6565e076888e6aab5e1fb86589a7 --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/logs/debug-internal.log @@ -0,0 +1,236 @@ +2024-08-12 07:32:02,368 INFO StreamThr :14458 [internal.py:wandb_internal():86] W&B internal server running at pid: 14458, started at: 2024-08-12 07:32:02.367023 +2024-08-12 07:32:02,369 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status +2024-08-12 07:32:02,371 INFO WriterThread:14458 [datastore.py:open_for_write():87] open: /project/wandb/run-20240812_073202-yby212na/run-yby212na.wandb +2024-08-12 07:32:02,372 DEBUG SenderThread:14458 [sender.py:send():382] send: header +2024-08-12 07:32:02,386 DEBUG SenderThread:14458 [sender.py:send():382] send: run +2024-08-12 07:32:02,917 INFO SenderThread:14458 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240812_073202-yby212na/files +2024-08-12 07:32:02,917 INFO SenderThread:14458 [sender.py:_start_run_threads():1136] run started: yby212na with start time 1723415522.366221 +2024-08-12 07:32:02,923 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: check_version +2024-08-12 07:32:02,923 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: check_version +2024-08-12 07:32:03,012 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: run_start +2024-08-12 07:32:03,018 DEBUG HandlerThread:14458 [system_info.py:__init__():27] System info init +2024-08-12 07:32:03,018 DEBUG HandlerThread:14458 [system_info.py:__init__():42] System info init done +2024-08-12 07:32:03,018 INFO HandlerThread:14458 [system_monitor.py:start():194] Starting system monitor +2024-08-12 07:32:03,019 INFO SystemMonitor:14458 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-12 07:32:03,019 INFO HandlerThread:14458 [system_monitor.py:probe():214] Collecting system info +2024-08-12 07:32:03,019 INFO SystemMonitor:14458 [interfaces.py:start():190] Started cpu monitoring +2024-08-12 07:32:03,020 INFO SystemMonitor:14458 [interfaces.py:start():190] Started disk monitoring +2024-08-12 07:32:03,020 INFO SystemMonitor:14458 [interfaces.py:start():190] Started gpu monitoring +2024-08-12 07:32:03,021 INFO SystemMonitor:14458 [interfaces.py:start():190] Started memory monitoring +2024-08-12 07:32:03,022 INFO SystemMonitor:14458 [interfaces.py:start():190] Started network monitoring +2024-08-12 07:32:03,032 DEBUG HandlerThread:14458 [system_info.py:probe():151] Probing system +2024-08-12 07:32:03,034 DEBUG HandlerThread:14458 [system_info.py:_probe_git():136] Probing git +2024-08-12 07:32:03,046 DEBUG HandlerThread:14458 [system_info.py:_probe_git():144] Probing git done +2024-08-12 07:32:03,047 DEBUG HandlerThread:14458 [system_info.py:probe():199] Probing system done +2024-08-12 07:32:03,047 DEBUG HandlerThread:14458 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-11T22:32:03.032279', 'startedAt': '2024-08-11T22:32:02.353340', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--valid-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--test-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '1', '--eval-interval', '5', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-qwen2-0.5B', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-qwen2-0.5B_train_2024-08-12-07:31:51'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0429999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}, {'current': 2400.043, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.487823486328125}} +2024-08-12 07:32:03,047 INFO HandlerThread:14458 [system_monitor.py:probe():224] Finished collecting system info +2024-08-12 07:32:03,047 INFO HandlerThread:14458 [system_monitor.py:probe():227] Publishing system info +2024-08-12 07:32:03,048 INFO HandlerThread:14458 [system_monitor.py:probe():229] Finished publishing system info +2024-08-12 07:32:03,054 DEBUG SenderThread:14458 [sender.py:send():382] send: files +2024-08-12 07:32:03,054 INFO SenderThread:14458 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-12 07:32:03,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: python_packages +2024-08-12 07:32:03,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:32:03,065 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:32:03,065 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: python_packages +2024-08-12 07:32:03,067 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:32:03,383 DEBUG SenderThread:14458 [sender.py:send():382] send: telemetry +2024-08-12 07:32:03,716 INFO wandb-upload_0:14458 [upload_job.py:push():131] Uploaded file /tmp/tmpjkv15ab8wandb/lrd2pdzk-wandb-metadata.json +2024-08-12 07:32:03,919 INFO Thread-12 :14458 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073202-yby212na/files/requirements.txt +2024-08-12 07:32:03,920 INFO Thread-12 :14458 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:32:03,920 INFO Thread-12 :14458 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073202-yby212na/files/wandb-metadata.json +2024-08-12 07:32:04,384 DEBUG SenderThread:14458 [sender.py:send():382] send: config +2024-08-12 07:32:04,384 DEBUG SenderThread:14458 [sender.py:send():382] send: config +2024-08-12 07:32:05,920 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:32:07,384 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:12,385 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:17,386 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:18,065 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:32:18,065 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:32:18,065 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:32:23,322 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:28,323 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:33,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:32:33,064 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:32:33,104 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:32:34,273 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:34,938 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/config.yaml +2024-08-12 07:32:39,667 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:44,667 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:48,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:32:48,064 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:32:48,108 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:32:50,338 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:32:55,338 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:00,339 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:03,022 DEBUG SystemMonitor:14458 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-12 07:33:03,024 DEBUG SenderThread:14458 [sender.py:send():382] send: stats +2024-08-12 07:33:03,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:33:03,064 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:33:03,104 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:33:06,281 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:11,281 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:16,282 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:18,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:33:18,064 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:33:18,108 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:33:19,954 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: partial_history +2024-08-12 07:33:21,450 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:21,967 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:33:23,969 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:33:25,970 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:33:27,344 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:32,345 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:33,025 DEBUG SenderThread:14458 [sender.py:send():382] send: stats +2024-08-12 07:33:33,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:33:33,065 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:33:33,066 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:33:38,333 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:43,334 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:48,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:33:48,065 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:33:48,104 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:33:49,288 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:54,289 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:33:59,290 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:03,026 DEBUG SenderThread:14458 [sender.py:send():382] send: stats +2024-08-12 07:34:03,064 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: stop_status +2024-08-12 07:34:03,065 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: stop_status +2024-08-12 07:34:03,108 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-12 07:34:05,251 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:10,252 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:15,187 DEBUG SenderThread:14458 [sender.py:send():382] send: exit +2024-08-12 07:34:15,187 INFO SenderThread:14458 [sender.py:send_exit():589] handling exit code: 255 +2024-08-12 07:34:15,187 INFO SenderThread:14458 [sender.py:send_exit():591] handling runtime: 132 +2024-08-12 07:34:15,189 INFO SenderThread:14458 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:34:15,189 INFO SenderThread:14458 [sender.py:send_exit():597] send defer +2024-08-12 07:34:15,189 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,190 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-12 07:34:15,190 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,190 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-12 07:34:15,190 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 1 +2024-08-12 07:34:15,190 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,190 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-12 07:34:15,190 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,190 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-12 07:34:15,190 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 2 +2024-08-12 07:34:15,190 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,190 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-12 07:34:15,190 INFO HandlerThread:14458 [system_monitor.py:finish():203] Stopping system monitor +2024-08-12 07:34:15,191 DEBUG SystemMonitor:14458 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-12 07:34:15,191 DEBUG SystemMonitor:14458 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-12 07:34:15,191 INFO HandlerThread:14458 [interfaces.py:finish():202] Joined cpu monitor +2024-08-12 07:34:15,192 INFO HandlerThread:14458 [interfaces.py:finish():202] Joined disk monitor +2024-08-12 07:34:15,225 INFO HandlerThread:14458 [interfaces.py:finish():202] Joined gpu monitor +2024-08-12 07:34:15,226 INFO HandlerThread:14458 [interfaces.py:finish():202] Joined memory monitor +2024-08-12 07:34:15,226 INFO HandlerThread:14458 [interfaces.py:finish():202] Joined network monitor +2024-08-12 07:34:15,226 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,226 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-12 07:34:15,226 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 3 +2024-08-12 07:34:15,227 DEBUG SenderThread:14458 [sender.py:send():382] send: stats +2024-08-12 07:34:15,227 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,227 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-12 07:34:15,229 DEBUG SenderThread:14458 [sender.py:send():382] send: history +2024-08-12 07:34:15,230 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: summary_record +2024-08-12 07:34:15,231 INFO SenderThread:14458 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:34:15,231 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,231 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-12 07:34:15,231 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 4 +2024-08-12 07:34:15,231 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,231 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-12 07:34:15,231 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,231 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-12 07:34:15,231 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 5 +2024-08-12 07:34:15,231 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,232 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-12 07:34:15,232 DEBUG SenderThread:14458 [sender.py:send():382] send: summary +2024-08-12 07:34:15,233 INFO SenderThread:14458 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-12 07:34:15,233 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,233 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-12 07:34:15,233 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 6 +2024-08-12 07:34:15,233 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,234 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-12 07:34:15,234 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,234 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-12 07:34:15,234 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 7 +2024-08-12 07:34:15,234 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:15,234 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,234 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-12 07:34:15,234 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,234 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-12 07:34:15,862 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 8 +2024-08-12 07:34:15,862 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,862 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-12 07:34:15,863 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:15,863 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-12 07:34:15,863 INFO SenderThread:14458 [job_builder.py:build():296] Attempting to build job artifact +2024-08-12 07:34:15,864 INFO SenderThread:14458 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-12 07:34:15,878 INFO SenderThread:14458 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-12 07:34:15,887 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 9 +2024-08-12 07:34:15,887 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:15,887 DEBUG SenderThread:14458 [sender.py:send():382] send: artifact +2024-08-12 07:34:15,887 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-12 07:34:16,002 INFO Thread-12 :14458 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:34:16,002 INFO Thread-12 :14458 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240812_073202-yby212na/files/wandb-summary.json +2024-08-12 07:34:16,187 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 07:34:16,750 INFO SenderThread:14458 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTEzOTg5OTc5MQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjM2MjY3MjMzNA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE0MDA5NDY1MQ==', 'versionIndex': 9}}} +2024-08-12 07:34:16,750 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:16,750 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-12 07:34:16,750 INFO SenderThread:14458 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-12 07:34:17,003 INFO SenderThread:14458 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240812_073202-yby212na/files +2024-08-12 07:34:17,004 INFO SenderThread:14458 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073202-yby212na/files/requirements.txt requirements.txt +2024-08-12 07:34:17,004 INFO SenderThread:14458 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073202-yby212na/files/config.yaml config.yaml +2024-08-12 07:34:17,004 INFO SenderThread:14458 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073202-yby212na/files/wandb-metadata.json wandb-metadata.json +2024-08-12 07:34:17,006 INFO SenderThread:14458 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073202-yby212na/files/wandb-summary.json wandb-summary.json +2024-08-12 07:34:17,008 INFO SenderThread:14458 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240812_073202-yby212na/files/output.log output.log +2024-08-12 07:34:17,009 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 10 +2024-08-12 07:34:17,009 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 07:34:17,011 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:17,011 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-12 07:34:17,012 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:17,012 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-12 07:34:17,012 INFO SenderThread:14458 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 07:34:17,188 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-12 07:34:17,188 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: poll_exit +2024-08-12 07:34:17,408 INFO wandb-upload_1:14458 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_073202-yby212na/files/config.yaml +2024-08-12 07:34:17,511 INFO wandb-upload_0:14458 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_073202-yby212na/files/requirements.txt +2024-08-12 07:34:17,588 INFO wandb-upload_2:14458 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_073202-yby212na/files/wandb-summary.json +2024-08-12 07:34:17,614 INFO wandb-upload_3:14458 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240812_073202-yby212na/files/output.log +2024-08-12 07:34:17,814 INFO Thread-11 (_thread_body):14458 [sender.py:transition_state():617] send defer: 11 +2024-08-12 07:34:17,814 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:17,815 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-12 07:34:17,815 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:17,815 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-12 07:34:17,815 INFO SenderThread:14458 [file_pusher.py:join():178] waiting for file pusher +2024-08-12 07:34:17,815 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 12 +2024-08-12 07:34:17,815 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:17,815 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-12 07:34:17,815 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:17,815 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-12 07:34:17,815 INFO SenderThread:14458 [file_stream.py:finish():595] file stream finish called +2024-08-12 07:34:18,362 INFO SenderThread:14458 [file_stream.py:finish():599] file stream finish is done +2024-08-12 07:34:18,362 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 13 +2024-08-12 07:34:18,362 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:18,363 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-12 07:34:18,363 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:18,363 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-12 07:34:18,363 INFO SenderThread:14458 [sender.py:transition_state():617] send defer: 14 +2024-08-12 07:34:18,363 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: defer +2024-08-12 07:34:18,363 DEBUG SenderThread:14458 [sender.py:send():382] send: final +2024-08-12 07:34:18,363 INFO HandlerThread:14458 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-12 07:34:18,363 DEBUG SenderThread:14458 [sender.py:send():382] send: footer +2024-08-12 07:34:18,364 DEBUG SenderThread:14458 [sender.py:send_request():409] send_request: defer +2024-08-12 07:34:18,364 INFO SenderThread:14458 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-12 07:34:21,364 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:26,365 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:31,366 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:36,367 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:41,367 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:46,368 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:51,369 DEBUG HandlerThread:14458 [handler.py:handle_request():146] handle_request: status_report +2024-08-12 07:34:51,550 WARNING StreamThr :14458 [internal.py:is_dead():414] Internal process exiting, parent pid 14387 disappeared +2024-08-12 07:34:51,550 ERROR StreamThr :14458 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-12 07:34:52,369 INFO SenderThread:14458 [sender.py:finish():1572] shutting down sender +2024-08-12 07:34:52,369 INFO SenderThread:14458 [file_pusher.py:finish():172] shutting down file pusher +2024-08-12 07:34:52,369 INFO SenderThread:14458 [file_pusher.py:join():178] waiting for file pusher +2024-08-12 07:34:52,369 INFO HandlerThread:14458 [handler.py:finish():869] shutting down handler +2024-08-12 07:34:52,369 INFO WriterThread:14458 [datastore.py:close():296] close: /project/wandb/run-20240812_073202-yby212na/run-yby212na.wandb diff --git a/wandb/run-20240812_073202-yby212na/logs/debug.log b/wandb/run-20240812_073202-yby212na/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..400f448f35d92fbb4842104f2cfc708b19eb524e --- /dev/null +++ b/wandb/run-20240812_073202-yby212na/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-12 07:32:02,359 INFO MainThread:14387 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-12 07:32:02,359 INFO MainThread:14387 [wandb_setup.py:_flush():76] Configure stats pid to 14387 +2024-08-12 07:32:02,359 INFO MainThread:14387 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'} +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240812_073202-yby212na/logs/debug.log +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240812_073202-yby212na/logs/debug-internal.log +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:init():566] calling init triggers +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-12-07:31:51', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 5, 'save_interval': 1, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320} +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:init():616] starting backend +2024-08-12 07:32:02,360 INFO MainThread:14387 [wandb_init.py:init():620] setting up manager +2024-08-12 07:32:02,365 INFO MainThread:14387 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-12 07:32:02,366 INFO MainThread:14387 [wandb_init.py:init():628] backend started and connected +2024-08-12 07:32:02,370 INFO MainThread:14387 [wandb_init.py:init():720] updated telemetry +2024-08-12 07:32:02,382 INFO MainThread:14387 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-12 07:32:02,922 INFO MainThread:14387 [wandb_run.py:_on_init():2262] communicating current version +2024-08-12 07:32:03,004 INFO MainThread:14387 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-12 07:32:03,004 INFO MainThread:14387 [wandb_init.py:init():804] starting run threads in backend +2024-08-12 07:32:03,064 INFO MainThread:14387 [wandb_run.py:_console_start():2241] atexit reg +2024-08-12 07:32:03,064 INFO MainThread:14387 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-12 07:32:03,064 INFO MainThread:14387 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-12 07:32:03,064 INFO MainThread:14387 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-12 07:32:03,065 INFO MainThread:14387 [wandb_init.py:init():847] run started, returning control to user process +2024-08-12 07:32:04,383 INFO MainThread:14387 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-12 07:32:04,383 INFO MainThread:14387 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240812_073202-yby212na/run-yby212na.wandb b/wandb/run-20240812_073202-yby212na/run-yby212na.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c4a0185d4a685a17a0bea57bb11c332fa584ba9c Binary files /dev/null and b/wandb/run-20240812_073202-yby212na/run-yby212na.wandb differ diff --git a/wandb/run-20240815_041534-1ld4rgmy/files/config.yaml b/wandb/run-20240815_041534-1ld4rgmy/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..042bf339ef01ac58d401eaf44e11289a9c7b5a14 --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/files/config.yaml @@ -0,0 +1,337 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +valid_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +test_data_path: + desc: null + value: + - '304771887' + - /work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 4096 +num_workers: + desc: null + value: 2 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-qwen2-0.5B_train_2024-08-15-04:15:21 +wandb_project: + desc: null + value: llm_tutorial +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 10 +save_interval: + desc: null + value: 10 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: adam +lr: + desc: null + value: 2.0e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 20000 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 1.0e-06 +train_iters: + desc: null + value: 20000 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 320 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 4096 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-06 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: false +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 320 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1723662934.646627 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 896 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 14 +num_hidden_layers: + desc: null + value: 24 diff --git a/wandb/run-20240815_041534-1ld4rgmy/files/output.log b/wandb/run-20240815_041534-1ld4rgmy/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..1398f04640f8c7aa12a16fadb9a1b908236fd799 --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/files/output.log @@ -0,0 +1,92 @@ +Created Hugging Face repository with ID koichi12/yans-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loaded model state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 6400000 + validation: 6403200 + test: 3200 +> building train, validation, and test datasets for GPT ... +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_init_utils.py:441: UserWarning: FSDP is switching to use `NO_SHARD` instead of ShardingStrategy.FULL_SHARD since the world size is 1. + warnings.warn( +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +Loaded optimizer state dict from /work/llm_recipes/models/yans-qwen2-0.5B/iter_0001160/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 896) + (layers): ModuleList( + (0-23): 24 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=896, out_features=896, bias=True) + (k_proj): Linear(in_features=896, out_features=128, bias=True) + (v_proj): Linear(in_features=896, out_features=128, bias=True) + (o_proj): Linear(in_features=896, out_features=896, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=896, out_features=4864, bias=False) + (up_proj): Linear(in_features=896, out_features=4864, bias=False) + (down_proj): Linear(in_features=4864, out_features=896, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=896, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 896, + "initializer_range": 0.02, + "intermediate_size": 4864, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "max_window_layers": 24, + "model_type": "qwen2", + "num_attention_heads": 14, + "num_hidden_layers": 24, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-15 04:15:41,598] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 71.0304706218284, Tokens per sec: 17664.9211934734, Loss: 2.442603349685669 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20240815_041534-1ld4rgmy/files/requirements.txt b/wandb/run-20240815_041534-1ld4rgmy/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8532a930d76c3129cd1689fa0e3e82842c9604cc --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/files/requirements.txt @@ -0,0 +1,354 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jedi==0.19.1 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240815_041534-1ld4rgmy/files/wandb-metadata.json b/wandb/run-20240815_041534-1ld4rgmy/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..eb1fe50e1c2706419e351df2d9a24d489677eabb --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/files/wandb-metadata.json @@ -0,0 +1,215 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-14T19:15:35.173102", + "startedAt": "2024-08-14T19:15:34.633818", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "4096", + "--sliding-window-size", + "4096", + "--micro-batch-size", + "1", + "--global-batch-size", + "320", + "--train-iters", + "20000", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--valid-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--test-data-path", + "304771887", + "/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document", + "--lr", + "2e-5", + "--min-lr", + "1e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "20000", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "adam", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-6", + "--save-interval", + "10", + "--eval-interval", + "10", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-qwen2-0.5B", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "llm_tutorial", + "--wandb-name", + "yans-qwen2-0.5B_train_2024-08-15-04:15:21" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "6da01327e78c302bc0cfdb335f3ca297e2a19c8c" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0389999999993, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20240815_041534-1ld4rgmy/files/wandb-summary.json b/wandb/run-20240815_041534-1ld4rgmy/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b1a8ff88f3fb87d22027409c8fd9325fa104b6a6 --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 86}, "training/loss": 2.442603349685669, "training/perplexity": 11.502947992429535, "utils/batch_size": 1, "utils/global_batch_size": 320, "utils/seq_len": 4097, "utils/gradient_accumulation_steps": 320, "utils/iteration": 1161, "optimizer/lr": 1.9946184158325198e-05, "optimizer/variance_l2": 0.0046823736576586325, "optimizer/variance_sqrt_l2": 0.5343142380105511, "optimizer/momentum_l2": 0.12459250428605805, "optimizer/weight_l2": 825.0639369164065, "optimizer/variance_l1": 0.284942626953125, "optimizer/variance_sqrt_l1": 4625.0, "optimizer/momentum_l1": 977.875, "optimizer/weight_l1": 6918144.0, "optimizer/variance_abs_max": 0.0030059814453125, "optimizer/variance_sqrt_abs_max": 0.054931640625, "optimizer/momentum_abs_max": 0.0108642578125, "optimizer/weight_abs_max": 175.0, "stats/1_iteration_time": 74.21714400200017, "stats/tokens_per_sec": 17664.9211934734, "stats/tokens_per_sec_per_gpu": 17664.9211934734, "stats/tflops": 71.0304706218284, "_timestamp": 1723663016.4553976, "_runtime": 81.8087706565857, "_step": 1161} \ No newline at end of file diff --git a/wandb/run-20240815_041534-1ld4rgmy/logs/debug-internal.log b/wandb/run-20240815_041534-1ld4rgmy/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..75f622f748249df2d5e31027202207e1ed972290 --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/logs/debug-internal.log @@ -0,0 +1,162 @@ +2024-08-15 04:15:34,649 INFO StreamThr :12253 [internal.py:wandb_internal():86] W&B internal server running at pid: 12253, started at: 2024-08-15 04:15:34.648066 +2024-08-15 04:15:34,650 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status +2024-08-15 04:15:34,652 INFO WriterThread:12253 [datastore.py:open_for_write():87] open: /project/wandb/run-20240815_041534-1ld4rgmy/run-1ld4rgmy.wandb +2024-08-15 04:15:34,653 DEBUG SenderThread:12253 [sender.py:send():382] send: header +2024-08-15 04:15:34,666 DEBUG SenderThread:12253 [sender.py:send():382] send: run +2024-08-15 04:15:35,078 INFO SenderThread:12253 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240815_041534-1ld4rgmy/files +2024-08-15 04:15:35,078 INFO SenderThread:12253 [sender.py:_start_run_threads():1136] run started: 1ld4rgmy with start time 1723662934.646627 +2024-08-15 04:15:35,084 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: check_version +2024-08-15 04:15:35,084 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: check_version +2024-08-15 04:15:35,155 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: run_start +2024-08-15 04:15:35,161 DEBUG HandlerThread:12253 [system_info.py:__init__():27] System info init +2024-08-15 04:15:35,161 DEBUG HandlerThread:12253 [system_info.py:__init__():42] System info init done +2024-08-15 04:15:35,161 INFO HandlerThread:12253 [system_monitor.py:start():194] Starting system monitor +2024-08-15 04:15:35,161 INFO SystemMonitor:12253 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-15 04:15:35,161 INFO HandlerThread:12253 [system_monitor.py:probe():214] Collecting system info +2024-08-15 04:15:35,162 INFO SystemMonitor:12253 [interfaces.py:start():190] Started cpu monitoring +2024-08-15 04:15:35,162 INFO SystemMonitor:12253 [interfaces.py:start():190] Started disk monitoring +2024-08-15 04:15:35,163 INFO SystemMonitor:12253 [interfaces.py:start():190] Started gpu monitoring +2024-08-15 04:15:35,164 INFO SystemMonitor:12253 [interfaces.py:start():190] Started memory monitoring +2024-08-15 04:15:35,164 INFO SystemMonitor:12253 [interfaces.py:start():190] Started network monitoring +2024-08-15 04:15:35,173 DEBUG HandlerThread:12253 [system_info.py:probe():151] Probing system +2024-08-15 04:15:35,175 DEBUG HandlerThread:12253 [system_info.py:_probe_git():136] Probing git +2024-08-15 04:15:35,188 DEBUG HandlerThread:12253 [system_info.py:_probe_git():144] Probing git done +2024-08-15 04:15:35,188 DEBUG HandlerThread:12253 [system_info.py:probe():199] Probing system done +2024-08-15 04:15:35,188 DEBUG HandlerThread:12253 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-14T19:15:35.173102', 'startedAt': '2024-08-14T19:15:34.633818', 'docker': None, 'cuda': None, 'args': ('--seq-length', '4096', '--sliding-window-size', '4096', '--micro-batch-size', '1', '--global-batch-size', '320', '--train-iters', '20000', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--valid-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--test-data-path', '304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document', '--lr', '2e-5', '--min-lr', '1e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '20000', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'adam', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-6', '--save-interval', '10', '--eval-interval', '10', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-qwen2-0.5B', '--fsdp-activation-checkpointing', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'llm_tutorial', '--wandb-name', 'yans-qwen2-0.5B_train_2024-08-15-04:15:21'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '6da01327e78c302bc0cfdb335f3ca297e2a19c8c'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0389999999993, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-08-15 04:15:35,188 INFO HandlerThread:12253 [system_monitor.py:probe():224] Finished collecting system info +2024-08-15 04:15:35,188 INFO HandlerThread:12253 [system_monitor.py:probe():227] Publishing system info +2024-08-15 04:15:35,189 INFO HandlerThread:12253 [system_monitor.py:probe():229] Finished publishing system info +2024-08-15 04:15:35,195 DEBUG SenderThread:12253 [sender.py:send():382] send: files +2024-08-15 04:15:35,195 INFO SenderThread:12253 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-15 04:15:35,207 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: python_packages +2024-08-15 04:15:35,207 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:15:35,207 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: python_packages +2024-08-15 04:15:35,208 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:15:35,209 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:15:35,448 DEBUG SenderThread:12253 [sender.py:send():382] send: telemetry +2024-08-15 04:15:35,826 INFO wandb-upload_0:12253 [upload_job.py:push():131] Uploaded file /tmp/tmprvuc38znwandb/8jb1h2yo-wandb-metadata.json +2024-08-15 04:15:36,080 INFO Thread-12 :12253 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_041534-1ld4rgmy/files/requirements.txt +2024-08-15 04:15:36,080 INFO Thread-12 :12253 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_041534-1ld4rgmy/files/wandb-metadata.json +2024-08-15 04:15:36,081 INFO Thread-12 :12253 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:15:38,081 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:15:40,019 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:15:40,082 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:15:41,878 DEBUG SenderThread:12253 [sender.py:send():382] send: config +2024-08-15 04:15:41,878 DEBUG SenderThread:12253 [sender.py:send():382] send: config +2024-08-15 04:15:42,083 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:15:44,084 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:15:45,879 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:15:50,206 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:15:50,206 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:15:50,208 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:15:51,411 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:15:56,411 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:01,412 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:05,206 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:16:05,206 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:16:05,246 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:16:06,461 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:08,114 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/config.yaml +2024-08-15 04:16:12,324 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:17,325 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:20,207 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:16:20,207 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:16:20,250 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:16:22,438 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:27,438 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:32,439 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:35,164 DEBUG SystemMonitor:12253 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-15 04:16:35,166 DEBUG SenderThread:12253 [sender.py:send():382] send: stats +2024-08-15 04:16:35,206 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:16:35,206 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:16:35,250 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:16:38,433 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:43,434 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:48,434 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:50,206 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: stop_status +2024-08-15 04:16:50,206 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: stop_status +2024-08-15 04:16:50,250 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-15 04:16:54,406 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:16:56,456 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: partial_history +2024-08-15 04:16:58,142 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:16:59,499 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:17:02,068 DEBUG SenderThread:12253 [sender.py:send():382] send: exit +2024-08-15 04:17:02,069 INFO SenderThread:12253 [sender.py:send_exit():589] handling exit code: 255 +2024-08-15 04:17:02,069 INFO SenderThread:12253 [sender.py:send_exit():591] handling runtime: 86 +2024-08-15 04:17:02,070 INFO SenderThread:12253 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 04:17:02,070 INFO SenderThread:12253 [sender.py:send_exit():597] send defer +2024-08-15 04:17:02,071 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,071 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-15 04:17:02,071 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,071 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-15 04:17:02,071 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 1 +2024-08-15 04:17:02,071 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,071 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-15 04:17:02,071 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,071 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-15 04:17:02,071 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 2 +2024-08-15 04:17:02,071 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,072 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-15 04:17:02,072 INFO HandlerThread:12253 [system_monitor.py:finish():203] Stopping system monitor +2024-08-15 04:17:02,072 DEBUG SystemMonitor:12253 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-15 04:17:02,072 INFO HandlerThread:12253 [interfaces.py:finish():202] Joined cpu monitor +2024-08-15 04:17:02,072 DEBUG SystemMonitor:12253 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-15 04:17:02,072 INFO HandlerThread:12253 [interfaces.py:finish():202] Joined disk monitor +2024-08-15 04:17:02,107 INFO HandlerThread:12253 [interfaces.py:finish():202] Joined gpu monitor +2024-08-15 04:17:02,107 INFO HandlerThread:12253 [interfaces.py:finish():202] Joined memory monitor +2024-08-15 04:17:02,107 INFO HandlerThread:12253 [interfaces.py:finish():202] Joined network monitor +2024-08-15 04:17:02,108 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,108 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-15 04:17:02,108 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 3 +2024-08-15 04:17:02,108 DEBUG SenderThread:12253 [sender.py:send():382] send: stats +2024-08-15 04:17:02,108 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,108 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-15 04:17:02,111 DEBUG SenderThread:12253 [sender.py:send():382] send: history +2024-08-15 04:17:02,111 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: summary_record +2024-08-15 04:17:02,112 INFO SenderThread:12253 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 04:17:02,113 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,113 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-15 04:17:02,113 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 4 +2024-08-15 04:17:02,113 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,113 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-15 04:17:02,113 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,113 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-15 04:17:02,113 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 5 +2024-08-15 04:17:02,113 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,113 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-15 04:17:02,114 DEBUG SenderThread:12253 [sender.py:send():382] send: summary +2024-08-15 04:17:02,115 INFO SenderThread:12253 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-15 04:17:02,115 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,115 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-15 04:17:02,115 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 6 +2024-08-15 04:17:02,115 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,115 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-15 04:17:02,115 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,115 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-15 04:17:02,116 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 7 +2024-08-15 04:17:02,116 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: status_report +2024-08-15 04:17:02,116 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:02,116 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-15 04:17:02,116 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:02,116 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-15 04:17:02,145 INFO Thread-12 :12253 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240815_041534-1ld4rgmy/files/wandb-summary.json +2024-08-15 04:17:03,068 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-15 04:17:03,854 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 8 +2024-08-15 04:17:03,854 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: poll_exit +2024-08-15 04:17:03,854 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:03,855 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-15 04:17:03,855 DEBUG SenderThread:12253 [sender.py:send_request():409] send_request: defer +2024-08-15 04:17:03,855 INFO SenderThread:12253 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-15 04:17:03,855 INFO SenderThread:12253 [job_builder.py:build():296] Attempting to build job artifact +2024-08-15 04:17:03,856 INFO SenderThread:12253 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-15 04:17:03,871 INFO SenderThread:12253 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-15 04:17:03,880 INFO SenderThread:12253 [sender.py:transition_state():617] send defer: 9 +2024-08-15 04:17:03,880 DEBUG SenderThread:12253 [sender.py:send():382] send: artifact +2024-08-15 04:17:03,880 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: defer +2024-08-15 04:17:03,881 INFO HandlerThread:12253 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-15 04:17:04,069 DEBUG HandlerThread:12253 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-15 04:17:04,146 INFO Thread-12 :12253 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240815_041534-1ld4rgmy/files/output.log +2024-08-15 04:17:35,760 WARNING StreamThr :12253 [internal.py:is_dead():414] Internal process exiting, parent pid 12182 disappeared +2024-08-15 04:17:35,760 ERROR StreamThr :12253 [internal.py:wandb_internal():152] Internal process shutdown. +2024-08-15 04:17:36,070 INFO WriterThread:12253 [datastore.py:close():296] close: /project/wandb/run-20240815_041534-1ld4rgmy/run-1ld4rgmy.wandb +2024-08-15 04:17:36,071 INFO HandlerThread:12253 [handler.py:finish():869] shutting down handler diff --git a/wandb/run-20240815_041534-1ld4rgmy/logs/debug.log b/wandb/run-20240815_041534-1ld4rgmy/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f70c9a4ffa26d7e23ad6e9d483fba038f7fbf682 --- /dev/null +++ b/wandb/run-20240815_041534-1ld4rgmy/logs/debug.log @@ -0,0 +1,29 @@ +2024-08-15 04:15:34,639 INFO MainThread:12182 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Configure stats pid to 12182 +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train Qwen2'} +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240815_041534-1ld4rgmy/logs/debug.log +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240815_041534-1ld4rgmy/logs/debug-internal.log +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:init():566] calling init triggers +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'valid_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'test_data_path': ['304771887', '/work/llm_recipes/datasets/bin/sample/llm_jp_corpus_v2_ja_wiki_train_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 4096, 'num_workers': 2, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-qwen2-0.5B_train_2024-08-15-04:15:21', 'wandb_project': 'llm_tutorial', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 10, 'save_interval': 10, 'eval_iters': 10, 'optimizer': 'adam', 'lr': 2e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 20000, 'lr_warmup_iters': 500, 'min_lr': 1e-06, 'train_iters': 20000, 'train_samples': None, 'global_batch_size': 320, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 4096, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-06, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': False, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 320} +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:init():616] starting backend +2024-08-15 04:15:34,640 INFO MainThread:12182 [wandb_init.py:init():620] setting up manager +2024-08-15 04:15:34,645 INFO MainThread:12182 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-15 04:15:34,646 INFO MainThread:12182 [wandb_init.py:init():628] backend started and connected +2024-08-15 04:15:34,651 INFO MainThread:12182 [wandb_init.py:init():720] updated telemetry +2024-08-15 04:15:34,662 INFO MainThread:12182 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-15 04:15:35,083 INFO MainThread:12182 [wandb_run.py:_on_init():2262] communicating current version +2024-08-15 04:15:35,107 INFO MainThread:12182 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.6 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-15 04:15:35,107 INFO MainThread:12182 [wandb_init.py:init():804] starting run threads in backend +2024-08-15 04:15:35,205 INFO MainThread:12182 [wandb_run.py:_console_start():2241] atexit reg +2024-08-15 04:15:35,206 INFO MainThread:12182 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-15 04:15:35,206 INFO MainThread:12182 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-15 04:15:35,206 INFO MainThread:12182 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-15 04:15:35,207 INFO MainThread:12182 [wandb_init.py:init():847] run started, returning control to user process +2024-08-15 04:15:41,877 INFO MainThread:12182 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 896, 'model_type': 'qwen2', 'max_position_embeddings': 4096, 'num_attention_heads': 14, 'num_hidden_layers': 24} +2024-08-15 04:15:41,877 INFO MainThread:12182 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} diff --git a/wandb/run-20240815_041534-1ld4rgmy/run-1ld4rgmy.wandb b/wandb/run-20240815_041534-1ld4rgmy/run-1ld4rgmy.wandb new file mode 100644 index 0000000000000000000000000000000000000000..4c9cbf2c990fd58c661283116ff354d6233a5d78 Binary files /dev/null and b/wandb/run-20240815_041534-1ld4rgmy/run-1ld4rgmy.wandb differ diff --git a/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml b/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b93022f5b779aabb0ea5dc37f7a0c69ecea8e9b6 --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml @@ -0,0 +1,321 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-0.5B_train_2024-08-24-20:20:07 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-0.5B +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-0.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-06 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-07 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-0.5B +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724498422.652614 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 diff --git a/wandb/run-20240824_202022-z2bjbf6e/files/output.log b/wandb/run-20240824_202022-z2bjbf6e/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..29ec19129308866369d74ef906daf4374a816b20 --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/files/output.log @@ -0,0 +1,51 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-0.5B. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-0.5B/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-0.5B, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-0.5B +--> /share/pretrained_lm/Qwen/Qwen2-0.5B has 494.032768 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Unable to save the indexes because path_to_cache is None +Traceback (most recent call last): + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 270, in build_generic_dataset + dataset = cls(*args) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 359, in __init__ + self.initialize(path_prefix, multimodal) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 374, in initialize + self.index = _IndexReader(get_idx_path(self.path_prefix), self.multimodal) + File "/project/megatron_lm/megatron/core/datasets/indexed_dataset.py", line 233, in __init__ + with open(idx_path, "rb") as stream: +FileNotFoundError: [Errno 2] No such file or directory: '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document.idx' +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 162, in main + train_dataset, validation_dataset, test_dataset = build_train_valid_test_datasets() + File "/project/src/llama_recipes/datasets/pretrain_dataset.py", line 76, in build_train_valid_test_datasets + return train_valid_test_datasets_provider(train_val_test_num_samples) + File "/project/src/llama_recipes/datasets/pretrain_dataset.py", line 46, in train_valid_test_datasets_provider + ).build() + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 56, in build + return self._build_blended_dataset_splits() + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 162, in _build_blended_dataset_splits + self._build_megatron_dataset_splits( + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 199, in _build_megatron_dataset_splits + indexed_dataset = self.build_generic_dataset( + File "/project/megatron_lm/megatron/core/datasets/blended_megatron_dataset_builder.py", line 278, in build_generic_dataset + raise Exception(log) from err +Exception: Failed to write dataset materials to the data cache directory. Please supply a directory to which you have write access via the path_to_cache attribute in BlendedMegatronDatasetConfig and retry. Refer to the preserved traceback above for more information. +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... \ No newline at end of file diff --git a/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt b/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240824_202022-z2bjbf6e/files/wandb-metadata.json b/wandb/run-20240824_202022-z2bjbf6e/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b25e123b9897f52daca2923a75955d7a7ecfd6af --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-24T11:20:23.248321", + "startedAt": "2024-08-24T11:20:22.637930", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-6", + "--min-lr", + "3.5e-7", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-0.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-0.5B", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-0.5B", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-0.5B_train_2024-08-24-20:20:07" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0340000000015, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.4449462890625 + } +} diff --git a/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json b/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b1d4cf96d64955a2ae8b982ce021e29fde546a1a --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 4}} \ No newline at end of file diff --git a/wandb/run-20240824_202022-z2bjbf6e/logs/debug-internal.log b/wandb/run-20240824_202022-z2bjbf6e/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..61fd526d908da3d1393d2bb95293d0d522418be1 --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/logs/debug-internal.log @@ -0,0 +1,191 @@ +2024-08-24 20:20:22,655 INFO StreamThr :25836 [internal.py:wandb_internal():86] W&B internal server running at pid: 25836, started at: 2024-08-24 20:20:22.654049 +2024-08-24 20:20:22,656 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: status +2024-08-24 20:20:22,659 INFO WriterThread:25836 [datastore.py:open_for_write():87] open: /project/wandb/run-20240824_202022-z2bjbf6e/run-z2bjbf6e.wandb +2024-08-24 20:20:22,660 DEBUG SenderThread:25836 [sender.py:send():382] send: header +2024-08-24 20:20:22,676 DEBUG SenderThread:25836 [sender.py:send():382] send: run +2024-08-24 20:20:23,101 INFO SenderThread:25836 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240824_202022-z2bjbf6e/files +2024-08-24 20:20:23,101 INFO SenderThread:25836 [sender.py:_start_run_threads():1136] run started: z2bjbf6e with start time 1724498422.652614 +2024-08-24 20:20:23,106 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: check_version +2024-08-24 20:20:23,106 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: check_version +2024-08-24 20:20:23,175 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: run_start +2024-08-24 20:20:23,182 DEBUG HandlerThread:25836 [system_info.py:__init__():27] System info init +2024-08-24 20:20:23,182 DEBUG HandlerThread:25836 [system_info.py:__init__():42] System info init done +2024-08-24 20:20:23,182 INFO HandlerThread:25836 [system_monitor.py:start():194] Starting system monitor +2024-08-24 20:20:23,182 INFO SystemMonitor:25836 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-24 20:20:23,183 INFO HandlerThread:25836 [system_monitor.py:probe():214] Collecting system info +2024-08-24 20:20:23,183 INFO SystemMonitor:25836 [interfaces.py:start():190] Started cpu monitoring +2024-08-24 20:20:23,183 INFO SystemMonitor:25836 [interfaces.py:start():190] Started disk monitoring +2024-08-24 20:20:23,184 INFO SystemMonitor:25836 [interfaces.py:start():190] Started gpu monitoring +2024-08-24 20:20:23,185 INFO SystemMonitor:25836 [interfaces.py:start():190] Started memory monitoring +2024-08-24 20:20:23,186 INFO SystemMonitor:25836 [interfaces.py:start():190] Started network monitoring +2024-08-24 20:20:23,248 DEBUG HandlerThread:25836 [system_info.py:probe():151] Probing system +2024-08-24 20:20:23,250 DEBUG HandlerThread:25836 [system_info.py:_probe_git():136] Probing git +2024-08-24 20:20:23,264 DEBUG HandlerThread:25836 [system_info.py:_probe_git():144] Probing git done +2024-08-24 20:20:23,264 DEBUG HandlerThread:25836 [system_info.py:probe():199] Probing system done +2024-08-24 20:20:23,264 DEBUG HandlerThread:25836 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-24T11:20:23.248321', 'startedAt': '2024-08-24T11:20:22.637930', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '16', '--valid_micro_batch_size', '1', '--global-batch-size', '1280', '--train-iters', '23178', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--test-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--lr', '3.5e-6', '--min-lr', '3.5e-7', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '23178', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-0.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-0.5B', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-0.5B_train_2024-08-24-20:20:07'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 144, 'cpu_count_logical': 144, 'cpu_freq': {'current': 2400.0340000000015, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}, {'current': 2400.034, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 453.4449462890625}} +2024-08-24 20:20:23,264 INFO HandlerThread:25836 [system_monitor.py:probe():224] Finished collecting system info +2024-08-24 20:20:23,264 INFO HandlerThread:25836 [system_monitor.py:probe():227] Publishing system info +2024-08-24 20:20:23,266 INFO HandlerThread:25836 [system_monitor.py:probe():229] Finished publishing system info +2024-08-24 20:20:23,272 DEBUG SenderThread:25836 [sender.py:send():382] send: files +2024-08-24 20:20:23,272 INFO SenderThread:25836 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-24 20:20:23,283 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: python_packages +2024-08-24 20:20:23,284 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: stop_status +2024-08-24 20:20:23,284 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-24 20:20:23,284 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: python_packages +2024-08-24 20:20:23,286 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: stop_status +2024-08-24 20:20:23,526 DEBUG SenderThread:25836 [sender.py:send():382] send: telemetry +2024-08-24 20:20:23,973 INFO wandb-upload_0:25836 [upload_job.py:push():131] Uploaded file /tmp/tmpwjpjqs3pwandb/55szr5f9-wandb-metadata.json +2024-08-24 20:20:24,103 INFO Thread-12 :25836 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:24,103 INFO Thread-12 :25836 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-metadata.json +2024-08-24 20:20:24,103 INFO Thread-12 :25836 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt +2024-08-24 20:20:26,103 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:27,701 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: status_report +2024-08-24 20:20:27,737 DEBUG SenderThread:25836 [sender.py:send():382] send: exit +2024-08-24 20:20:27,737 INFO SenderThread:25836 [sender.py:send_exit():589] handling exit code: 1 +2024-08-24 20:20:27,737 INFO SenderThread:25836 [sender.py:send_exit():591] handling runtime: 4 +2024-08-24 20:20:27,739 INFO SenderThread:25836 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-24 20:20:27,739 INFO SenderThread:25836 [sender.py:send_exit():597] send defer +2024-08-24 20:20:27,739 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:27,739 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-24 20:20:27,740 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:27,740 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-24 20:20:27,740 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 1 +2024-08-24 20:20:27,740 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:27,740 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-24 20:20:27,740 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:27,740 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-24 20:20:27,740 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 2 +2024-08-24 20:20:27,740 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:27,740 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-24 20:20:27,740 INFO HandlerThread:25836 [system_monitor.py:finish():203] Stopping system monitor +2024-08-24 20:20:27,740 DEBUG SystemMonitor:25836 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-24 20:20:27,741 INFO HandlerThread:25836 [interfaces.py:finish():202] Joined cpu monitor +2024-08-24 20:20:27,741 DEBUG SystemMonitor:25836 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-24 20:20:27,741 INFO HandlerThread:25836 [interfaces.py:finish():202] Joined disk monitor +2024-08-24 20:20:27,741 DEBUG SystemMonitor:25836 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-24 20:20:28,105 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:28,106 INFO Thread-12 :25836 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json +2024-08-24 20:20:28,918 INFO HandlerThread:25836 [interfaces.py:finish():202] Joined gpu monitor +2024-08-24 20:20:28,918 INFO HandlerThread:25836 [interfaces.py:finish():202] Joined memory monitor +2024-08-24 20:20:28,918 INFO HandlerThread:25836 [interfaces.py:finish():202] Joined network monitor +2024-08-24 20:20:28,918 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:28,920 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:28,920 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-24 20:20:28,920 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 3 +2024-08-24 20:20:28,920 DEBUG SenderThread:25836 [sender.py:send():382] send: stats +2024-08-24 20:20:28,920 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:28,921 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:28,921 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-24 20:20:28,921 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:28,921 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-24 20:20:28,921 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 4 +2024-08-24 20:20:28,921 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:28,921 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-24 20:20:28,922 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:28,922 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-24 20:20:28,922 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 5 +2024-08-24 20:20:28,922 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:28,922 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-24 20:20:28,922 DEBUG SenderThread:25836 [sender.py:send():382] send: summary +2024-08-24 20:20:28,923 INFO SenderThread:25836 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-24 20:20:28,923 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:28,923 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-24 20:20:28,923 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 6 +2024-08-24 20:20:28,923 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:28,924 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-24 20:20:28,924 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:28,924 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-24 20:20:28,927 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: status_report +2024-08-24 20:20:29,107 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json +2024-08-24 20:20:29,126 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 7 +2024-08-24 20:20:29,126 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:29,126 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-24 20:20:29,126 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:29,126 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-24 20:20:29,738 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:30,108 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml +2024-08-24 20:20:30,108 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:31,391 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 8 +2024-08-24 20:20:31,392 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:31,392 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:31,392 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-24 20:20:31,392 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:31,392 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-24 20:20:31,392 INFO SenderThread:25836 [job_builder.py:build():296] Attempting to build job artifact +2024-08-24 20:20:31,393 INFO SenderThread:25836 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-24 20:20:31,408 INFO SenderThread:25836 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-24 20:20:31,417 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 9 +2024-08-24 20:20:31,418 DEBUG SenderThread:25836 [sender.py:send():382] send: artifact +2024-08-24 20:20:31,418 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:31,419 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-24 20:20:31,738 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:32,109 INFO Thread-12 :25836 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:34,782 INFO SenderThread:25836 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2MzU1Mzg0Mw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2MzU1Mzg0Mw==', 'versionIndex': 0}}} +2024-08-24 20:20:34,782 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:34,782 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: status_report +2024-08-24 20:20:34,782 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-24 20:20:34,783 INFO SenderThread:25836 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-24 20:20:35,110 INFO SenderThread:25836 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240824_202022-z2bjbf6e/files +2024-08-24 20:20:35,110 INFO SenderThread:25836 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt requirements.txt +2024-08-24 20:20:35,110 INFO SenderThread:25836 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml config.yaml +2024-08-24 20:20:35,112 INFO SenderThread:25836 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-metadata.json wandb-metadata.json +2024-08-24 20:20:35,112 INFO SenderThread:25836 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json wandb-summary.json +2024-08-24 20:20:35,113 INFO SenderThread:25836 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log output.log +2024-08-24 20:20:35,115 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 10 +2024-08-24 20:20:35,115 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:35,116 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:35,117 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-24 20:20:35,117 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:35,117 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-24 20:20:35,117 INFO SenderThread:25836 [file_pusher.py:finish():172] shutting down file pusher +2024-08-24 20:20:35,574 INFO wandb-upload_1:25836 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202022-z2bjbf6e/files/config.yaml +2024-08-24 20:20:35,574 INFO wandb-upload_0:25836 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202022-z2bjbf6e/files/requirements.txt +2024-08-24 20:20:35,580 INFO wandb-upload_2:25836 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202022-z2bjbf6e/files/wandb-summary.json +2024-08-24 20:20:35,588 INFO wandb-upload_3:25836 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240824_202022-z2bjbf6e/files/output.log +2024-08-24 20:20:35,739 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:35,740 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:35,788 INFO Thread-11 (_thread_body):25836 [sender.py:transition_state():617] send defer: 11 +2024-08-24 20:20:35,788 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:35,788 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-24 20:20:35,789 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:35,789 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-24 20:20:35,789 INFO SenderThread:25836 [file_pusher.py:join():178] waiting for file pusher +2024-08-24 20:20:35,789 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 12 +2024-08-24 20:20:35,789 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:35,789 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-24 20:20:35,789 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:35,789 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-24 20:20:35,789 INFO SenderThread:25836 [file_stream.py:finish():595] file stream finish called +2024-08-24 20:20:36,056 INFO SenderThread:25836 [file_stream.py:finish():599] file stream finish is done +2024-08-24 20:20:36,056 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 13 +2024-08-24 20:20:36,056 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:36,056 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-24 20:20:36,056 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:36,056 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-24 20:20:36,056 INFO SenderThread:25836 [sender.py:transition_state():617] send defer: 14 +2024-08-24 20:20:36,057 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: defer +2024-08-24 20:20:36,057 DEBUG SenderThread:25836 [sender.py:send():382] send: final +2024-08-24 20:20:36,057 INFO HandlerThread:25836 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-24 20:20:36,057 DEBUG SenderThread:25836 [sender.py:send():382] send: footer +2024-08-24 20:20:36,057 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: defer +2024-08-24 20:20:36,057 INFO SenderThread:25836 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-24 20:20:36,057 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:36,057 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:36,058 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-24 20:20:36,058 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: server_info +2024-08-24 20:20:36,058 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: poll_exit +2024-08-24 20:20:36,058 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: server_info +2024-08-24 20:20:36,060 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: get_summary +2024-08-24 20:20:36,060 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-24 20:20:36,060 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-24 20:20:36,060 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: job_info +2024-08-24 20:20:36,224 DEBUG SenderThread:25836 [sender.py:send_request():409] send_request: job_info +2024-08-24 20:20:36,224 INFO MainThread:25836 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-24 20:20:36,224 INFO MainThread:25836 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-24 20:20:36,224 INFO MainThread:25836 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-24 20:20:36,225 DEBUG HandlerThread:25836 [handler.py:handle_request():146] handle_request: shutdown +2024-08-24 20:20:36,225 INFO HandlerThread:25836 [handler.py:finish():869] shutting down handler +2024-08-24 20:20:37,061 INFO WriterThread:25836 [datastore.py:close():296] close: /project/wandb/run-20240824_202022-z2bjbf6e/run-z2bjbf6e.wandb +2024-08-24 20:20:37,224 INFO SenderThread:25836 [sender.py:finish():1572] shutting down sender +2024-08-24 20:20:37,224 INFO SenderThread:25836 [file_pusher.py:finish():172] shutting down file pusher +2024-08-24 20:20:37,224 INFO SenderThread:25836 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240824_202022-z2bjbf6e/logs/debug.log b/wandb/run-20240824_202022-z2bjbf6e/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..70f94d74ea9b35b03dac71d7d57413f9a3760793 --- /dev/null +++ b/wandb/run-20240824_202022-z2bjbf6e/logs/debug.log @@ -0,0 +1,28 @@ +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Configure stats pid to 25210 +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train baseline'} +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240824_202022-z2bjbf6e/logs/debug.log +2024-08-24 20:20:22,645 INFO MainThread:25210 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240824_202022-z2bjbf6e/logs/debug-internal.log +2024-08-24 20:20:22,646 INFO MainThread:25210 [wandb_init.py:init():566] calling init triggers +2024-08-24 20:20:22,646 INFO MainThread:25210 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-0.5B_train_2024-08-24-20:20:07', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-0.5B', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-0.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-06, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-07, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-0.5B', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-24 20:20:22,646 INFO MainThread:25210 [wandb_init.py:init():616] starting backend +2024-08-24 20:20:22,646 INFO MainThread:25210 [wandb_init.py:init():620] setting up manager +2024-08-24 20:20:22,651 INFO MainThread:25210 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-24 20:20:22,652 INFO MainThread:25210 [wandb_init.py:init():628] backend started and connected +2024-08-24 20:20:22,659 INFO MainThread:25210 [wandb_init.py:init():720] updated telemetry +2024-08-24 20:20:22,672 INFO MainThread:25210 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-24 20:20:23,105 INFO MainThread:25210 [wandb_run.py:_on_init():2262] communicating current version +2024-08-24 20:20:23,127 INFO MainThread:25210 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-24 20:20:23,127 INFO MainThread:25210 [wandb_init.py:init():804] starting run threads in backend +2024-08-24 20:20:23,283 INFO MainThread:25210 [wandb_run.py:_console_start():2241] atexit reg +2024-08-24 20:20:23,283 INFO MainThread:25210 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-24 20:20:23,283 INFO MainThread:25210 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-24 20:20:23,283 INFO MainThread:25210 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-24 20:20:23,284 INFO MainThread:25210 [wandb_init.py:init():847] run started, returning control to user process +2024-08-24 20:20:37,225 WARNING MsgRouterThr:25210 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20240824_202022-z2bjbf6e/run-z2bjbf6e.wandb b/wandb/run-20240824_202022-z2bjbf6e/run-z2bjbf6e.wandb new file mode 100644 index 0000000000000000000000000000000000000000..a006c9218043dce59af2ba3188561c3ba8f3ef7b Binary files /dev/null and b/wandb/run-20240824_202022-z2bjbf6e/run-z2bjbf6e.wandb differ diff --git a/wandb/run-20240826_221726-7jzdp89j/files/config.yaml b/wandb/run-20240826_221726-7jzdp89j/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d5832b4491b20818dfc47f85fdeaa7ecfd49cd5 --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-1.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-26-22:17:00 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5 +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-1.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-06 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-1.5B-3.5e-5 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724678246.995911 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 1536 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 12 +num_hidden_layers: + desc: null + value: 28 diff --git a/wandb/run-20240826_221726-7jzdp89j/files/output.log b/wandb/run-20240826_221726-7jzdp89j/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..73d767789f871a7d518f92781708a6b78794972b --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/files/output.log @@ -0,0 +1,29089 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-1.5B-3.5e-5. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +File not found: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5, skipping model loading +--> Model /share/pretrained_lm/Qwen/Qwen2-1.5B +--> /share/pretrained_lm/Qwen/Qwen2-1.5B has 1543.714304 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... +Let split = None +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5, skipping optimizer loading +File not found: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 1536) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=1536, out_features=1536, bias=True) + (k_proj): Linear(in_features=1536, out_features=256, bias=True) + (v_proj): Linear(in_features=1536, out_features=256, bias=True) + (o_proj): Linear(in_features=1536, out_features=1536, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=1536, out_features=8960, bias=False) + (up_proj): Linear(in_features=1536, out_features=8960, bias=False) + (down_proj): Linear(in_features=8960, out_features=1536, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=1536, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 85.08554552950555, Tokens per sec: 69525.58954895519, Loss: 4.733519554138184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 102.21094639894977, Tokens per sec: 83519.19544641528, Loss: 4.682239532470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 102.74810029672476, Tokens per sec: 83958.1177238587, Loss: 4.6787285804748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 103.44171467552798, Tokens per sec: 84524.88788800135, Loss: 4.656540870666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 103.43529183473868, Tokens per sec: 84519.63961945378, Loss: 4.646810054779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 101.85021967869476, Tokens per sec: 83224.43635736304, Loss: 4.636817455291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 103.47916680688985, Tokens per sec: 84555.49098864077, Loss: 4.604264259338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 103.4491091010667, Tokens per sec: 84530.9300634585, Loss: 4.533711910247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 103.42926877146134, Tokens per sec: 84514.71801940243, Loss: 4.503419876098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 101.93087755977312, Tokens per sec: 83290.34398831073, Loss: 4.494256973266602 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 103.47652389514667, Tokens per sec: 84553.33139742081, Loss: 4.453122138977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 102.68878471996278, Tokens per sec: 83909.6494391677, Loss: 4.408321380615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 102.5308178307126, Tokens per sec: 83780.57062753188, Loss: 4.386159420013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 103.40332416042354, Tokens per sec: 84493.51800984974, Loss: 4.3809638023376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 102.00761502328314, Tokens per sec: 83353.0481451428, Loss: 4.340020179748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 103.42154788571742, Tokens per sec: 84508.40908490778, Loss: 4.325527191162109 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 101.8676520566545, Tokens per sec: 83238.68080214328, Loss: 4.31596040725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 103.47923855196267, Tokens per sec: 84555.54961338665, Loss: 4.300565719604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 101.52658972098027, Tokens per sec: 82959.98998793826, Loss: 4.2624592781066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 103.43234662513798, Tokens per sec: 84517.23301287262, Loss: 4.281608581542969 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 101.8248958088301, Tokens per sec: 83203.74357140221, Loss: 4.252829074859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 103.40986279953925, Tokens per sec: 84498.86090019073, Loss: 4.2319536209106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 101.9796316360138, Tokens per sec: 83330.18219904858, Loss: 4.190131187438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 102.10547546867241, Tokens per sec: 83433.01243422237, Loss: 4.201891899108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 103.48068675496705, Tokens per sec: 84556.7329773416, Loss: 4.195281028747559 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 102.62907806687018, Tokens per sec: 83860.86159593974, Loss: 4.2009077072143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 103.47842633945683, Tokens per sec: 84554.88593363992, Loss: 4.179379940032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 103.33104650190785, Tokens per sec: 84434.45807448412, Loss: 4.170253276824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 103.4064504906315, Tokens per sec: 84496.07261474157, Loss: 4.177191257476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 103.45258283642949, Tokens per sec: 84533.76854204532, Loss: 4.142404556274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 103.43173604090985, Tokens per sec: 84516.73408878199, Loss: 4.144074440002441 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 103.37406441761588, Tokens per sec: 84469.60912078882, Loss: 4.107631206512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 102.5725195999256, Tokens per sec: 83814.64621665487, Loss: 4.113345146179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 102.55705304083223, Tokens per sec: 83802.00809307501, Loss: 4.076366424560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 103.40636984920815, Tokens per sec: 84496.00672055865, Loss: 4.118751525878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 103.40948966368185, Tokens per sec: 84498.55600127629, Loss: 4.097280025482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 103.41233930039206, Tokens per sec: 84500.88451278837, Loss: 4.064814567565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 102.64632849826215, Tokens per sec: 83874.95736749574, Loss: 4.062085151672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 103.32999511302636, Tokens per sec: 84433.5989575639, Loss: 4.055406093597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 102.48097184061383, Tokens per sec: 83739.84018587225, Loss: 4.049161911010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 102.82728545839777, Tokens per sec: 84022.8219578691, Loss: 4.041066646575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 103.3602438349966, Tokens per sec: 84458.31596695715, Loss: 4.047442436218262 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 103.44561343894242, Tokens per sec: 84528.07366795039, Loss: 3.985931873321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 102.42612837542208, Tokens per sec: 83695.02617866767, Loss: 4.024754047393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 102.58199526261168, Tokens per sec: 83822.38902456092, Loss: 4.005437850952148 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 103.35210911019787, Tokens per sec: 84451.66887391768, Loss: 4.024487018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 102.74523584026858, Tokens per sec: 83955.77710275067, Loss: 3.995710849761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 102.64869973654673, Tokens per sec: 83876.89496733916, Loss: 3.9599783420562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 103.52369707386134, Tokens per sec: 84591.87781609423, Loss: 3.990359306335449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 102.80812218050345, Tokens per sec: 84007.163150195, Loss: 3.9686155319213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 51 , TFLOPS: 102.57894278050884, Tokens per sec: 83819.8947628568, Loss: 3.9116101264953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 52 , TFLOPS: 102.71058226592926, Tokens per sec: 83927.46077509566, Loss: 3.9427168369293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 53 , TFLOPS: 102.5617661281954, Tokens per sec: 83805.85927808513, Loss: 3.913032293319702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 54 , TFLOPS: 102.74939982337884, Tokens per sec: 83959.17960054052, Loss: 3.9091131687164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 55 , TFLOPS: 102.5998328203102, Tokens per sec: 83836.96455213595, Loss: 3.9230899810791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 56 , TFLOPS: 102.7904818136628, Tokens per sec: 83992.74875234606, Loss: 3.909801959991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 57 , TFLOPS: 102.14188801019303, Tokens per sec: 83462.76605924108, Loss: 3.889183521270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 58 , TFLOPS: 103.44425671489532, Tokens per sec: 84526.96505380631, Loss: 3.899632692337036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 59 , TFLOPS: 101.63678160739605, Tokens per sec: 83050.0306149201, Loss: 3.8660147190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 60 , TFLOPS: 103.31274980841114, Tokens per sec: 84419.50737522918, Loss: 3.8469760417938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 61 , TFLOPS: 102.75221238039384, Tokens per sec: 83961.47781327914, Loss: 3.8679399490356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 62 , TFLOPS: 102.71313770459138, Tokens per sec: 83929.5488898093, Loss: 3.8520004749298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 63 , TFLOPS: 103.41666062995687, Tokens per sec: 84504.4155824167, Loss: 3.8401167392730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 64 , TFLOPS: 102.43621108099744, Tokens per sec: 83703.26501694522, Loss: 3.796480655670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 65 , TFLOPS: 103.43246849765826, Tokens per sec: 84517.33259804621, Loss: 3.7816340923309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 66 , TFLOPS: 103.38191344775606, Tokens per sec: 84476.02276536837, Loss: 3.8035836219787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 67 , TFLOPS: 103.42716469377207, Tokens per sec: 84512.99872335876, Loss: 3.801767349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 68 , TFLOPS: 102.6700973409269, Tokens per sec: 83894.37950070154, Loss: 3.7941091060638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 69 , TFLOPS: 103.295545306802, Tokens per sec: 84405.44913408106, Loss: 3.782762050628662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 70 , TFLOPS: 103.3459998636179, Tokens per sec: 84446.67685127117, Loss: 3.7403461933135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 71 , TFLOPS: 103.37106844818946, Tokens per sec: 84467.16103704738, Loss: 3.7470285892486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 72 , TFLOPS: 102.63176722245353, Tokens per sec: 83863.05897419194, Loss: 3.7460241317749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 73 , TFLOPS: 103.40552666793916, Tokens per sec: 84495.31773543835, Loss: 3.7371160984039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 74 , TFLOPS: 102.69843229472359, Tokens per sec: 83917.53272085542, Loss: 3.7013087272644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 75 , TFLOPS: 103.37748822742836, Tokens per sec: 84472.4068039233, Loss: 3.68121337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 76 , TFLOPS: 103.37740440296098, Tokens per sec: 84472.33830879316, Loss: 3.6894173622131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 77 , TFLOPS: 103.31263820251257, Tokens per sec: 84419.41617917645, Loss: 3.6768507957458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 78 , TFLOPS: 102.49514067602544, Tokens per sec: 83751.41790602534, Loss: 3.680011749267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 79 , TFLOPS: 102.79477162289749, Tokens per sec: 83996.25406590123, Loss: 3.6597535610198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 80 , TFLOPS: 103.38294547091853, Tokens per sec: 84476.86605806107, Loss: 3.6523332595825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 81 , TFLOPS: 103.35744437367099, Tokens per sec: 84456.02845504346, Loss: 3.646977186203003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 82 , TFLOPS: 102.64390918648897, Tokens per sec: 83872.98048556723, Loss: 3.6315174102783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 83 , TFLOPS: 102.59389627552076, Tokens per sec: 83832.11364856825, Loss: 3.6264429092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 84 , TFLOPS: 103.43911817106256, Tokens per sec: 84522.76621736269, Loss: 3.612766742706299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 85 , TFLOPS: 103.37870681777595, Tokens per sec: 84473.40254546561, Loss: 3.6021220684051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 86 , TFLOPS: 101.95422175052963, Tokens per sec: 83309.41912751082, Loss: 3.6046714782714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 87 , TFLOPS: 103.33342129772504, Tokens per sec: 84436.39858127903, Loss: 3.5712661743164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 88 , TFLOPS: 103.30100251548643, Tokens per sec: 84409.90835977816, Loss: 3.5666086673736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 89 , TFLOPS: 101.66439514331196, Tokens per sec: 83072.59434595265, Loss: 3.5666921138763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 90 , TFLOPS: 103.3179117578529, Tokens per sec: 84423.72533699842, Loss: 3.531656265258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 91 , TFLOPS: 101.96337300458197, Tokens per sec: 83316.89685277121, Loss: 3.567232131958008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 92 , TFLOPS: 103.34484181624458, Tokens per sec: 84445.73058095164, Loss: 3.534730911254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 93 , TFLOPS: 102.69629211602195, Tokens per sec: 83915.78392574521, Loss: 3.5254483222961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 94 , TFLOPS: 102.80987988751203, Tokens per sec: 84008.5994178391, Loss: 3.5332751274108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 95 , TFLOPS: 102.32646617294654, Tokens per sec: 83613.58962749157, Loss: 3.507675886154175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 96 , TFLOPS: 102.5410310250229, Tokens per sec: 83788.91609151383, Loss: 3.492846965789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 97 , TFLOPS: 101.99852117871859, Tokens per sec: 83345.61733065273, Loss: 3.50849986076355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 98 , TFLOPS: 103.4633897716942, Tokens per sec: 84542.59916704538, Loss: 3.4512650966644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 99 , TFLOPS: 101.68230977120861, Tokens per sec: 83087.23284957054, Loss: 3.4911234378814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 100 , TFLOPS: 102.63545132724948, Tokens per sec: 83866.06935105799, Loss: 3.4753124713897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 101 , TFLOPS: 102.74531547352676, Tokens per sec: 83955.84217313588, Loss: 3.461775779724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 102 , TFLOPS: 102.68186745799895, Tokens per sec: 83903.99716635088, Loss: 3.4674668312072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 103 , TFLOPS: 103.41052724601805, Tokens per sec: 84499.40383651294, Loss: 3.46848464012146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 104 , TFLOPS: 103.3844795286713, Tokens per sec: 84478.11957614102, Loss: 3.442750930786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 105 , TFLOPS: 103.31426257143733, Tokens per sec: 84420.74349283984, Loss: 3.4365429878234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 106 , TFLOPS: 103.38253766221172, Tokens per sec: 84476.53282706879, Loss: 3.4481163024902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 107 , TFLOPS: 103.43397209539306, Tokens per sec: 84518.56122646136, Loss: 3.4227805137634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 108 , TFLOPS: 103.2674171497396, Tokens per sec: 84382.46489286219, Loss: 3.3930890560150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 109 , TFLOPS: 103.3939072742852, Tokens per sec: 84485.82322977389, Loss: 3.409421682357788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 110 , TFLOPS: 101.05500674849577, Tokens per sec: 82574.6474014955, Loss: 3.386138916015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 111 , TFLOPS: 103.27258507171831, Tokens per sec: 84386.68773494515, Loss: 3.390360116958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 112 , TFLOPS: 103.29578341046586, Tokens per sec: 84405.64369471413, Loss: 3.382711172103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 113 , TFLOPS: 103.36384504682658, Tokens per sec: 84461.25860984687, Loss: 3.3799519538879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 114 , TFLOPS: 102.57703958244906, Tokens per sec: 83818.33961072948, Loss: 3.3509225845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 115 , TFLOPS: 103.31545957518269, Tokens per sec: 84421.7215954138, Loss: 3.3581318855285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 116 , TFLOPS: 102.50500980387699, Tokens per sec: 83759.4822244468, Loss: 3.3564789295196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 117 , TFLOPS: 103.34772971266366, Tokens per sec: 84448.09035545675, Loss: 3.357633590698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 118 , TFLOPS: 102.57940486134147, Tokens per sec: 83820.27234100016, Loss: 3.3455770015716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 119 , TFLOPS: 103.37674199953705, Tokens per sec: 84471.79704190369, Loss: 3.349000930786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 120 , TFLOPS: 101.95151624672962, Tokens per sec: 83307.20839070996, Loss: 3.363442897796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 121 , TFLOPS: 102.58678771676776, Tokens per sec: 83826.3050622206, Loss: 3.346540927886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 122 , TFLOPS: 103.27506375677196, Tokens per sec: 84388.71313230951, Loss: 3.325683116912842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 123 , TFLOPS: 102.63880956829217, Tokens per sec: 83868.81345626278, Loss: 3.3384294509887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 124 , TFLOPS: 102.02059954186137, Tokens per sec: 83363.65812952435, Loss: 3.299971103668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 125 , TFLOPS: 102.60096061366573, Tokens per sec: 83837.88610111875, Loss: 3.32597017288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 126 , TFLOPS: 103.35993622950448, Tokens per sec: 84458.06461459109, Loss: 3.2988574504852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 127 , TFLOPS: 101.93974000768604, Tokens per sec: 83297.58572264008, Loss: 3.3006503582000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 128 , TFLOPS: 102.57444595186637, Tokens per sec: 83816.22028841438, Loss: 3.2919490337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 129 , TFLOPS: 102.03075032984248, Tokens per sec: 83371.95259968821, Loss: 3.257246971130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 130 , TFLOPS: 102.5917719688214, Tokens per sec: 83830.3778228797, Loss: 3.2692110538482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 131 , TFLOPS: 101.89433505193749, Tokens per sec: 83260.4841644704, Loss: 3.2538375854492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 132 , TFLOPS: 102.681838315305, Tokens per sec: 83903.97335310545, Loss: 3.2454559803009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 133 , TFLOPS: 102.05735983182879, Tokens per sec: 83393.69590875076, Loss: 3.2827978134155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 134 , TFLOPS: 102.77393367568189, Tokens per sec: 83979.2268428149, Loss: 3.2658843994140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 135 , TFLOPS: 101.92537400891918, Tokens per sec: 83285.84689523398, Loss: 3.259977102279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 136 , TFLOPS: 103.32458682323742, Tokens per sec: 84429.17970475559, Loss: 3.242375135421753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 137 , TFLOPS: 102.0903627071719, Tokens per sec: 83420.66340776322, Loss: 3.2340521812438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 138 , TFLOPS: 102.68404719219257, Tokens per sec: 83905.77828327184, Loss: 3.2384321689605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 139 , TFLOPS: 102.81964712441989, Tokens per sec: 84016.58047854758, Loss: 3.2448322772979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 140 , TFLOPS: 102.79413398611031, Tokens per sec: 83995.73303646821, Loss: 3.2270736694335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 141 , TFLOPS: 102.78256949354912, Tokens per sec: 83986.28339189994, Loss: 3.235694169998169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 142 , TFLOPS: 101.87039070158662, Tokens per sec: 83240.91861941629, Loss: 3.19096302986145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 143 , TFLOPS: 103.366503153969, Tokens per sec: 84463.43061761871, Loss: 3.1855406761169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 144 , TFLOPS: 103.33311295409933, Tokens per sec: 84436.14662576499, Loss: 3.1958141326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 145 , TFLOPS: 103.38304693766686, Tokens per sec: 84476.9489691531, Loss: 3.1991422176361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 146 , TFLOPS: 102.68014364149673, Tokens per sec: 83902.58859151193, Loss: 3.2032370567321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 147 , TFLOPS: 103.24725363841142, Tokens per sec: 84365.98876870099, Loss: 3.1853559017181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 148 , TFLOPS: 102.61907187301703, Tokens per sec: 83852.68527736, Loss: 3.173830986022949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 149 , TFLOPS: 102.63767054999349, Tokens per sec: 83867.88273509, Loss: 3.1731185913085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 150 , TFLOPS: 103.44914988505847, Tokens per sec: 84530.96338910804, Loss: 3.2084667682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 151 , TFLOPS: 102.48653155972671, Tokens per sec: 83744.38317547928, Loss: 3.1839139461517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 152 , TFLOPS: 101.93195533658036, Tokens per sec: 83291.22466748438, Loss: 3.156155586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 153 , TFLOPS: 103.34626885845883, Tokens per sec: 84446.89665388023, Loss: 3.159332752227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 154 , TFLOPS: 103.29826343524843, Tokens per sec: 84407.6701868055, Loss: 3.155775547027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 155 , TFLOPS: 102.51115759439968, Tokens per sec: 83764.50574234108, Loss: 3.142470598220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 156 , TFLOPS: 102.66173119754937, Tokens per sec: 83887.54330958407, Loss: 3.151242733001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 157 , TFLOPS: 101.92595457945504, Tokens per sec: 83286.32129436404, Loss: 3.139582633972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 158 , TFLOPS: 103.38773816333415, Tokens per sec: 84480.78229040874, Loss: 3.140867233276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 159 , TFLOPS: 101.86810745428524, Tokens per sec: 83239.05291927033, Loss: 3.1397194862365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 160 , TFLOPS: 103.37144571248915, Tokens per sec: 84467.4693094183, Loss: 3.117032766342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 161 , TFLOPS: 101.87509465281899, Tokens per sec: 83244.76233905874, Loss: 3.1112842559814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 162 , TFLOPS: 102.03635749315998, Tokens per sec: 83376.53435717619, Loss: 3.1471879482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 163 , TFLOPS: 102.29582658545854, Tokens per sec: 83588.55323182199, Loss: 3.147831439971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 164 , TFLOPS: 102.75179412592526, Tokens per sec: 83961.13604678604, Loss: 3.1009318828582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 165 , TFLOPS: 101.79102311231935, Tokens per sec: 83176.0653191225, Loss: 3.1005797386169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 166 , TFLOPS: 102.79351015734476, Tokens per sec: 83995.2232899251, Loss: 3.150515079498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 167 , TFLOPS: 102.63555899499228, Tokens per sec: 83866.15732914224, Loss: 3.098729372024536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 168 , TFLOPS: 103.29933788416581, Tokens per sec: 84408.54814667467, Loss: 3.1161341667175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 169 , TFLOPS: 101.91364546017614, Tokens per sec: 83276.26319613634, Loss: 3.1012070178985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 170 , TFLOPS: 102.643745621388, Tokens per sec: 83872.8468323127, Loss: 3.109105348587036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 171 , TFLOPS: 103.35090586195602, Tokens per sec: 84450.68566880489, Loss: 3.094001531600952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 172 , TFLOPS: 102.12963286449158, Tokens per sec: 83452.7520642129, Loss: 3.0737009048461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 173 , TFLOPS: 102.65145546253733, Tokens per sec: 83879.14674198478, Loss: 3.1004855632781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 174 , TFLOPS: 101.57145040537092, Tokens per sec: 82996.64680797062, Loss: 3.1030373573303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 175 , TFLOPS: 103.45685225449843, Tokens per sec: 84537.25719345406, Loss: 3.0763370990753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 176 , TFLOPS: 103.28327533744663, Tokens per sec: 84395.42302626367, Loss: 3.086561679840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 177 , TFLOPS: 102.56369529320882, Tokens per sec: 83807.43564847873, Loss: 3.070050001144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 178 , TFLOPS: 102.657954858316, Tokens per sec: 83884.45756558487, Loss: 3.063524007797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 179 , TFLOPS: 102.75526455007362, Tokens per sec: 83963.97181969433, Loss: 3.0781900882720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 180 , TFLOPS: 100.48385203206598, Tokens per sec: 82107.94217987033, Loss: 3.0302979946136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 181 , TFLOPS: 102.53580042350958, Tokens per sec: 83784.64203236965, Loss: 3.057932138442993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 182 , TFLOPS: 102.75729635070164, Tokens per sec: 83965.63205629069, Loss: 3.0501091480255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 183 , TFLOPS: 103.40325516413205, Tokens per sec: 84493.46163120384, Loss: 3.0621824264526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 184 , TFLOPS: 100.58670639280797, Tokens per sec: 82191.98712573938, Loss: 3.0514042377471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 185 , TFLOPS: 103.28190883674172, Tokens per sec: 84394.30642336085, Loss: 3.0571231842041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 186 , TFLOPS: 102.61245438917352, Tokens per sec: 83847.27796096227, Loss: 3.0424506664276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 187 , TFLOPS: 101.98492259700163, Tokens per sec: 83334.50557947317, Loss: 3.046173334121704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 188 , TFLOPS: 102.68854839510233, Tokens per sec: 83909.45633204066, Loss: 3.042165756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 189 , TFLOPS: 102.7703317798024, Tokens per sec: 83976.28364097071, Loss: 3.0116958618164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 190 , TFLOPS: 101.27875704765329, Tokens per sec: 82757.47953078222, Loss: 3.0313944816589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 191 , TFLOPS: 102.70127747623253, Tokens per sec: 83919.85759190786, Loss: 3.0335030555725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 192 , TFLOPS: 102.53350373819038, Tokens per sec: 83782.76535167338, Loss: 3.0446932315826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 193 , TFLOPS: 100.9298115197791, Tokens per sec: 82472.34715729921, Loss: 3.0333523750305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 194 , TFLOPS: 102.10458472487146, Tokens per sec: 83432.28458453283, Loss: 3.035475969314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 195 , TFLOPS: 102.09048908442124, Tokens per sec: 83420.76667386688, Loss: 3.0086820125579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 196 , TFLOPS: 102.8236874932487, Tokens per sec: 84019.88196792593, Loss: 3.0406062602996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 197 , TFLOPS: 100.21208323573893, Tokens per sec: 81885.87290044028, Loss: 3.0226449966430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 198 , TFLOPS: 103.35583892530938, Tokens per sec: 84454.71660185928, Loss: 2.9923787117004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 199 , TFLOPS: 101.29302913670679, Tokens per sec: 82769.14162214409, Loss: 3.015338182449341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 200 , TFLOPS: 102.75537559958227, Tokens per sec: 83964.06256110652, Loss: 2.998582363128662 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/optimizer.pt +[rank0]:[2024-08-26 23:10:39,399] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00784134199784603, 'preprocessing_with_comm': 0.0022156659979373217, 'state_converting': 3.2753734959987924, : 3.2871911990077933}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200, took 15.56s + eval ppl=15.36564826965332, eval loss=2.7321343421936035 +------------------------------------------------------------------ +iteration: 201 , TFLOPS: 101.17047361744943, Tokens per sec: 82668.99835249917, Loss: 2.9920401573181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 202 , TFLOPS: 102.46502915351215, Tokens per sec: 83726.81300584017, Loss: 2.9941837787628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 203 , TFLOPS: 100.74750103871087, Tokens per sec: 82323.37657012907, Loss: 3.0173959732055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 204 , TFLOPS: 101.2202665973181, Tokens per sec: 82709.68547813519, Loss: 3.0006651878356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 205 , TFLOPS: 103.2932601457731, Tokens per sec: 84403.58187017908, Loss: 2.9996259212493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 206 , TFLOPS: 100.72251273719337, Tokens per sec: 82302.957985703, Loss: 2.968914270401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 207 , TFLOPS: 102.77601489044619, Tokens per sec: 83980.92745697404, Loss: 2.9629266262054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 208 , TFLOPS: 101.25866584478547, Tokens per sec: 82741.06249172536, Loss: 2.9853835105895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 209 , TFLOPS: 102.21086340169748, Tokens per sec: 83519.1276272239, Loss: 2.9823031425476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 210 , TFLOPS: 101.22465047680662, Tokens per sec: 82713.26765891635, Loss: 3.001328229904175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 211 , TFLOPS: 103.23656961991419, Tokens per sec: 84357.2585821558, Loss: 2.972769260406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 212 , TFLOPS: 101.71583475229376, Tokens per sec: 83114.62697462502, Loss: 2.9791438579559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 213 , TFLOPS: 102.61213610299828, Tokens per sec: 83847.01788113506, Loss: 2.996415138244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 214 , TFLOPS: 101.79997141856629, Tokens per sec: 83183.37721050676, Loss: 2.983034133911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 215 , TFLOPS: 102.12146927018905, Tokens per sec: 83446.08138115868, Loss: 2.9214422702789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 216 , TFLOPS: 102.65776427216804, Tokens per sec: 83884.30183273724, Loss: 2.9577088356018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 217 , TFLOPS: 101.41117395607527, Tokens per sec: 82865.6807953683, Loss: 2.95973801612854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 218 , TFLOPS: 103.35915682086237, Tokens per sec: 84457.42773972584, Loss: 2.969614267349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 219 , TFLOPS: 102.68672793363349, Tokens per sec: 83907.96878611164, Loss: 2.9661130905151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 220 , TFLOPS: 102.75501761458821, Tokens per sec: 83963.77004235251, Loss: 2.9539098739624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 221 , TFLOPS: 102.76226712227327, Tokens per sec: 83969.69380170092, Loss: 2.9316060543060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 222 , TFLOPS: 102.618629175998, Tokens per sec: 83852.3235382297, Loss: 2.93827486038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 223 , TFLOPS: 103.34547631257281, Tokens per sec: 84446.24904423482, Loss: 2.960069179534912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 224 , TFLOPS: 102.7608884610172, Tokens per sec: 83968.56726209867, Loss: 2.9648540019989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 225 , TFLOPS: 100.54406238022673, Tokens per sec: 82157.14160530471, Loss: 2.9608311653137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 226 , TFLOPS: 103.3784945165542, Tokens per sec: 84473.2290686722, Loss: 2.9652366638183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 227 , TFLOPS: 102.69569588585794, Tokens per sec: 83915.29673072987, Loss: 2.912226915359497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 228 , TFLOPS: 101.94229589207609, Tokens per sec: 83299.67420156952, Loss: 2.918229818344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 229 , TFLOPS: 102.65747089546193, Tokens per sec: 83884.06210707848, Loss: 2.905651569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 230 , TFLOPS: 102.76590376042284, Tokens per sec: 83972.66539234832, Loss: 2.9356935024261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 231 , TFLOPS: 101.0862836237126, Tokens per sec: 82600.20454137358, Loss: 2.925503969192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 232 , TFLOPS: 102.1988300551843, Tokens per sec: 83509.29487002209, Loss: 2.9362993240356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 233 , TFLOPS: 102.0210239997269, Tokens per sec: 83364.00496497279, Loss: 2.9223670959472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 234 , TFLOPS: 103.30362024475043, Tokens per sec: 84412.04737374613, Loss: 2.9233078956604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 235 , TFLOPS: 101.47590453937173, Tokens per sec: 82918.57382129328, Loss: 2.9182167053222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 236 , TFLOPS: 102.55504231090653, Tokens per sec: 83800.36507389194, Loss: 2.93105411529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 237 , TFLOPS: 103.34664923455195, Tokens per sec: 84447.20746897768, Loss: 2.894244909286499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 238 , TFLOPS: 101.4518769877653, Tokens per sec: 82898.94029035147, Loss: 2.9293432235717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 239 , TFLOPS: 102.13184898324884, Tokens per sec: 83454.56291189745, Loss: 2.8743088245391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 240 , TFLOPS: 102.71430176414655, Tokens per sec: 83930.50007283741, Loss: 2.913327217102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 241 , TFLOPS: 101.38085696592918, Tokens per sec: 82840.90800229117, Loss: 2.9035322666168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 242 , TFLOPS: 102.02818156027756, Tokens per sec: 83369.85359195073, Loss: 2.895106792449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 243 , TFLOPS: 102.7064803986711, Tokens per sec: 83924.10903376756, Loss: 2.891068696975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 244 , TFLOPS: 101.15771482242386, Tokens per sec: 82658.57281264318, Loss: 2.889805793762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 245 , TFLOPS: 102.81377736367564, Tokens per sec: 84011.78414594237, Loss: 2.8898983001708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 246 , TFLOPS: 102.03138312808767, Tokens per sec: 83372.46967542404, Loss: 2.9001359939575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 247 , TFLOPS: 102.28591545514992, Tokens per sec: 83580.45459211162, Loss: 2.9052205085754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 248 , TFLOPS: 101.46268578876493, Tokens per sec: 82907.77243989156, Loss: 2.893223762512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 249 , TFLOPS: 103.29688650320477, Tokens per sec: 84406.54506018724, Loss: 2.906332492828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 250 , TFLOPS: 101.82646140358057, Tokens per sec: 83205.02285917482, Loss: 2.8997278213500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 251 , TFLOPS: 102.75311795799692, Tokens per sec: 83962.21778405026, Loss: 2.90319561958313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 252 , TFLOPS: 96.02477723822484, Tokens per sec: 78464.3173790258, Loss: 2.9000041484832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 253 , TFLOPS: 102.15813574888963, Tokens per sec: 83476.04250478407, Loss: 2.856813907623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 254 , TFLOPS: 103.3349348058316, Tokens per sec: 84437.63530771405, Loss: 2.9007701873779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 255 , TFLOPS: 101.44088982963876, Tokens per sec: 82889.96240061166, Loss: 2.9071457386016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 256 , TFLOPS: 103.34577628214868, Tokens per sec: 84446.49415709717, Loss: 2.8727104663848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 257 , TFLOPS: 102.59040823520145, Tokens per sec: 83829.26348103321, Loss: 2.8822855949401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 258 , TFLOPS: 101.9451275003121, Tokens per sec: 83301.98798155124, Loss: 2.879142999649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 259 , TFLOPS: 103.31059509687474, Tokens per sec: 84417.74670496557, Loss: 2.88511061668396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 260 , TFLOPS: 102.68754518211229, Tokens per sec: 83908.6365808815, Loss: 2.856894016265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 261 , TFLOPS: 103.27602269554797, Tokens per sec: 84389.49670586866, Loss: 2.8596479892730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 262 , TFLOPS: 102.70579477458664, Tokens per sec: 83923.5487926786, Loss: 2.861494541168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 263 , TFLOPS: 100.54066757852738, Tokens per sec: 82154.3676254462, Loss: 2.862962245941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 264 , TFLOPS: 103.35316828051987, Tokens per sec: 84452.53434925324, Loss: 2.860217809677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 265 , TFLOPS: 102.85555506920653, Tokens per sec: 84045.92178458547, Loss: 2.864945411682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 266 , TFLOPS: 101.88629045924007, Tokens per sec: 83253.9107206911, Loss: 2.854949951171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 267 , TFLOPS: 102.65736452081228, Tokens per sec: 83883.97518561395, Loss: 2.8298771381378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 268 , TFLOPS: 102.7907322100309, Tokens per sec: 83992.9533576642, Loss: 2.8482422828674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 269 , TFLOPS: 101.63553808336695, Tokens per sec: 83049.01449942478, Loss: 2.8441176414489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 270 , TFLOPS: 101.43113289864924, Tokens per sec: 82881.98976113436, Loss: 2.874752998352051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 271 , TFLOPS: 102.62104471196825, Tokens per sec: 83854.29733485242, Loss: 2.859210968017578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 272 , TFLOPS: 102.62507562675185, Tokens per sec: 83857.59109908708, Loss: 2.8525614738464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 273 , TFLOPS: 101.28361833240443, Tokens per sec: 82761.45181169297, Loss: 2.862574338912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 274 , TFLOPS: 103.32013335295285, Tokens per sec: 84425.54065954355, Loss: 2.8501882553100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 275 , TFLOPS: 102.72520833973401, Tokens per sec: 83939.41211651015, Loss: 2.851802110671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 276 , TFLOPS: 102.06948785804727, Tokens per sec: 83403.60603117706, Loss: 2.8407483100891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 277 , TFLOPS: 102.03807544737427, Tokens per sec: 83377.93814178876, Loss: 2.845700979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 278 , TFLOPS: 102.85827365381391, Tokens per sec: 84048.14321005077, Loss: 2.8381402492523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 279 , TFLOPS: 101.1746787315396, Tokens per sec: 82672.43445947152, Loss: 2.847154378890991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 280 , TFLOPS: 101.81531047384281, Tokens per sec: 83195.91114743563, Loss: 2.8005826473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 281 , TFLOPS: 102.63194562154881, Tokens per sec: 83863.20474868506, Loss: 2.8357176780700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 282 , TFLOPS: 101.06005850028365, Tokens per sec: 82578.77531792503, Loss: 2.825500249862671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 283 , TFLOPS: 102.1518865463882, Tokens per sec: 83470.93612055137, Loss: 2.8433661460876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 284 , TFLOPS: 101.3949588331405, Tokens per sec: 82852.43100100389, Loss: 2.838899612426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 285 , TFLOPS: 102.21203863038991, Tokens per sec: 83520.08793684156, Loss: 2.8328418731689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 286 , TFLOPS: 101.43205599063535, Tokens per sec: 82882.74404335873, Loss: 2.8243393898010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 287 , TFLOPS: 103.30289383882352, Tokens per sec: 84411.4538087639, Loss: 2.797614097595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 288 , TFLOPS: 101.90381478978375, Tokens per sec: 83268.23030229475, Loss: 2.8182456493377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 289 , TFLOPS: 102.81344239572624, Tokens per sec: 84011.51043500807, Loss: 2.819993019104004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 290 , TFLOPS: 101.87725705609095, Tokens per sec: 83246.52929443747, Loss: 2.8266549110412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 291 , TFLOPS: 102.31130607061449, Tokens per sec: 83601.2019176203, Loss: 2.81638240814209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 292 , TFLOPS: 103.36825435236058, Tokens per sec: 84464.86156690432, Loss: 2.8240675926208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 293 , TFLOPS: 100.57304552805734, Tokens per sec: 82180.8244814897, Loss: 2.813002586364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 294 , TFLOPS: 103.41221397198818, Tokens per sec: 84500.78210372332, Loss: 2.8205814361572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 295 , TFLOPS: 102.85761410419539, Tokens per sec: 84047.60427507914, Loss: 2.8382325172424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 296 , TFLOPS: 102.71822643029758, Tokens per sec: 83933.7070185793, Loss: 2.8036932945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 297 , TFLOPS: 103.3182414914744, Tokens per sec: 84423.99477082866, Loss: 2.8047401905059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 298 , TFLOPS: 102.67491617191374, Tokens per sec: 83898.31709154863, Loss: 2.8002073764801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 299 , TFLOPS: 102.6130172706731, Tokens per sec: 83847.73790592543, Loss: 2.8066468238830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 300 , TFLOPS: 102.72950307412688, Tokens per sec: 83942.92145453887, Loss: 2.801112651824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 301 , TFLOPS: 101.30736185614391, Tokens per sec: 82780.85325615319, Loss: 2.7955422401428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 302 , TFLOPS: 103.31278192285046, Tokens per sec: 84419.53361676422, Loss: 2.799838066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 303 , TFLOPS: 102.78256253244061, Tokens per sec: 83986.27770379897, Loss: 2.788813829421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 304 , TFLOPS: 101.9373460123591, Tokens per sec: 83295.62952743152, Loss: 2.8034110069274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 305 , TFLOPS: 102.71442664190022, Tokens per sec: 83930.60211366457, Loss: 2.7823877334594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 306 , TFLOPS: 102.66268458288545, Tokens per sec: 83888.32234528451, Loss: 2.7876410484313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 307 , TFLOPS: 101.9148619666672, Tokens per sec: 83277.25723490585, Loss: 2.7804553508758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 308 , TFLOPS: 101.53814562737847, Tokens per sec: 82969.43261653172, Loss: 2.810614585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 309 , TFLOPS: 102.74798751409877, Tokens per sec: 83958.02556627171, Loss: 2.797494411468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 310 , TFLOPS: 103.47482480125703, Tokens per sec: 84551.94302406503, Loss: 2.786207675933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 311 , TFLOPS: 100.98880849987415, Tokens per sec: 82520.55510845224, Loss: 2.792884588241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 312 , TFLOPS: 103.37801777375903, Tokens per sec: 84472.83950986182, Loss: 2.777301549911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 313 , TFLOPS: 103.4160392804236, Tokens per sec: 84503.90786171811, Loss: 2.802597999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 314 , TFLOPS: 101.28977309781885, Tokens per sec: 82766.48102895086, Loss: 2.7956254482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 315 , TFLOPS: 102.64285736429316, Tokens per sec: 83872.12101457473, Loss: 2.7841246128082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 316 , TFLOPS: 101.94914099512438, Tokens per sec: 83305.26751147889, Loss: 2.758450746536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 317 , TFLOPS: 101.99094214245083, Tokens per sec: 83339.42430501661, Loss: 2.7866358757019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 318 , TFLOPS: 101.92834806545632, Tokens per sec: 83288.2770733895, Loss: 2.7851665019989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 319 , TFLOPS: 102.70928269605773, Tokens per sec: 83926.39886309972, Loss: 2.777468204498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 320 , TFLOPS: 101.25185122948574, Tokens per sec: 82735.49409414014, Loss: 2.7867918014526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 321 , TFLOPS: 101.96243549002841, Tokens per sec: 83316.13078549573, Loss: 2.774831533432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 322 , TFLOPS: 101.38524011337167, Tokens per sec: 82844.48958489868, Loss: 2.7581751346588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 323 , TFLOPS: 102.1132666458378, Tokens per sec: 83439.37880564695, Loss: 2.7818548679351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 324 , TFLOPS: 100.74513825021855, Tokens per sec: 82321.44587482841, Loss: 2.7837891578674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 325 , TFLOPS: 103.28183854758939, Tokens per sec: 84394.24898828506, Loss: 2.752873659133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 326 , TFLOPS: 101.2758621397133, Tokens per sec: 82755.11402698289, Loss: 2.7763609886169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 327 , TFLOPS: 102.66485644596939, Tokens per sec: 83890.09703051911, Loss: 2.7623443603515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 328 , TFLOPS: 101.9784479691366, Tokens per sec: 83329.21499437292, Loss: 2.746570110321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 329 , TFLOPS: 102.04547012953299, Tokens per sec: 83383.98052693669, Loss: 2.763117790222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 330 , TFLOPS: 103.3166833052909, Tokens per sec: 84422.72153678682, Loss: 2.7414798736572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 331 , TFLOPS: 101.53505902454536, Tokens per sec: 82966.91047390057, Loss: 2.761845111846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 332 , TFLOPS: 102.55591809636918, Tokens per sec: 83801.0807007382, Loss: 2.7629528045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 333 , TFLOPS: 103.31460065127685, Tokens per sec: 84421.01974657967, Loss: 2.7588307857513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 334 , TFLOPS: 102.57213813206481, Tokens per sec: 83814.33450944597, Loss: 2.7479918003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 335 , TFLOPS: 102.46948104090751, Tokens per sec: 83730.4507576327, Loss: 2.7546348571777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 336 , TFLOPS: 102.18381710180044, Tokens per sec: 83497.02739934434, Loss: 2.7403600215911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 337 , TFLOPS: 103.29750859484689, Tokens per sec: 84407.05338728208, Loss: 2.7556984424591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 338 , TFLOPS: 102.60246616232565, Tokens per sec: 83839.11632368506, Loss: 2.756244421005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 339 , TFLOPS: 102.07839157867183, Tokens per sec: 83410.88149050155, Loss: 2.74111270904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 340 , TFLOPS: 102.64660612118169, Tokens per sec: 83875.18422032993, Loss: 2.750070571899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 341 , TFLOPS: 101.93286155913415, Tokens per sec: 83291.96516526131, Loss: 2.755415916442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 342 , TFLOPS: 102.61264793516392, Tokens per sec: 83847.43611237334, Loss: 2.742366075515747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 343 , TFLOPS: 102.01666525288992, Tokens per sec: 83360.44332072815, Loss: 2.752568244934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 344 , TFLOPS: 102.6741494238173, Tokens per sec: 83897.69056194118, Loss: 2.77097487449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 345 , TFLOPS: 102.01843321429597, Tokens per sec: 83361.8879675044, Loss: 2.7608237266540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 346 , TFLOPS: 101.57961985793358, Tokens per sec: 83003.32227795999, Loss: 2.707113027572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 347 , TFLOPS: 102.66580202498413, Tokens per sec: 83890.86968747345, Loss: 2.765608549118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 348 , TFLOPS: 102.75362470065778, Tokens per sec: 83962.63185652316, Loss: 2.741978645324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 349 , TFLOPS: 101.24972304466465, Tokens per sec: 82733.75509953848, Loss: 2.7501096725463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 350 , TFLOPS: 102.66981310459896, Tokens per sec: 83894.14724388109, Loss: 2.74472713470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 351 , TFLOPS: 102.68479398655037, Tokens per sec: 83906.3885081658, Loss: 2.7408101558685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 352 , TFLOPS: 102.52414938832092, Tokens per sec: 83775.12167159266, Loss: 2.742112398147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 353 , TFLOPS: 101.33357992966108, Tokens per sec: 82802.27671893731, Loss: 2.742185354232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 354 , TFLOPS: 102.54791428661444, Tokens per sec: 83794.54058175121, Loss: 2.742248058319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 355 , TFLOPS: 101.52501614880491, Tokens per sec: 82958.70418160643, Loss: 2.735707998275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 356 , TFLOPS: 101.3463506133318, Tokens per sec: 82812.7119733113, Loss: 2.7373311519622803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 357 , TFLOPS: 103.2148902786747, Tokens per sec: 84339.54383435311, Loss: 2.7287721633911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 358 , TFLOPS: 100.99943801093706, Tokens per sec: 82529.24075556971, Loss: 2.7219791412353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 359 , TFLOPS: 102.63353773281214, Tokens per sec: 83864.50570378294, Loss: 2.7238612174987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 360 , TFLOPS: 102.02107136647129, Tokens per sec: 83364.04366955895, Loss: 2.741600751876831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 361 , TFLOPS: 102.20153600924164, Tokens per sec: 83511.50597473979, Loss: 2.7289915084838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 362 , TFLOPS: 101.17274222868531, Tokens per sec: 82670.85209313926, Loss: 2.726491928100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 363 , TFLOPS: 103.31082024776848, Tokens per sec: 84417.93068155673, Loss: 2.726564407348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 364 , TFLOPS: 101.3381102652185, Tokens per sec: 82805.9785727425, Loss: 2.7331509590148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 365 , TFLOPS: 102.6843878912193, Tokens per sec: 83906.05667721695, Loss: 2.7343411445617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 366 , TFLOPS: 102.13743444046939, Tokens per sec: 83459.1269327748, Loss: 2.7106471061706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 367 , TFLOPS: 102.05635098398658, Tokens per sec: 83392.87155320884, Loss: 2.7271969318389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 368 , TFLOPS: 103.43568797894446, Tokens per sec: 84519.96331908199, Loss: 2.718400478363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 369 , TFLOPS: 102.10486412612292, Tokens per sec: 83432.51289048782, Loss: 2.719304084777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 370 , TFLOPS: 102.6698534012108, Tokens per sec: 83894.18017127995, Loss: 2.718219757080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 371 , TFLOPS: 102.7067901515494, Tokens per sec: 83924.36214081816, Loss: 2.7164969444274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 372 , TFLOPS: 102.57616818077528, Tokens per sec: 83817.62756598997, Loss: 2.7211577892303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 373 , TFLOPS: 103.23505388082147, Tokens per sec: 84356.02003272454, Loss: 2.718363046646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 374 , TFLOPS: 102.03439882286726, Tokens per sec: 83374.93387725935, Loss: 2.734165668487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 375 , TFLOPS: 103.28651875851722, Tokens per sec: 84398.07330911355, Loss: 2.7232794761657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 376 , TFLOPS: 101.97283960416036, Tokens per sec: 83324.63225497866, Loss: 2.6980204582214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 377 , TFLOPS: 102.09778330376118, Tokens per sec: 83426.72696825946, Loss: 2.7203688621520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 378 , TFLOPS: 102.73201339047004, Tokens per sec: 83944.97269864415, Loss: 2.696538209915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 379 , TFLOPS: 102.6061389644241, Tokens per sec: 83842.11746482603, Loss: 2.709707736968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 380 , TFLOPS: 102.65498649630683, Tokens per sec: 83882.03204057475, Loss: 2.703300952911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 381 , TFLOPS: 102.23583847199357, Tokens per sec: 83539.53540007833, Loss: 2.703430414199829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 382 , TFLOPS: 102.02008018699237, Tokens per sec: 83363.2337512915, Loss: 2.713082790374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 383 , TFLOPS: 101.8918589435689, Tokens per sec: 83258.46087258174, Loss: 2.7146518230438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 384 , TFLOPS: 102.171906675975, Tokens per sec: 83487.29508380071, Loss: 2.714290142059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 385 , TFLOPS: 102.56082498031006, Tokens per sec: 83805.09024192073, Loss: 2.69993257522583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 386 , TFLOPS: 102.65849096053378, Tokens per sec: 83884.89562850751, Loss: 2.7126126289367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 387 , TFLOPS: 101.29910888479, Tokens per sec: 82774.10954080953, Loss: 2.683380365371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 388 , TFLOPS: 102.7821275163959, Tokens per sec: 83985.92224099055, Loss: 2.7132656574249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 389 , TFLOPS: 102.65978157230678, Tokens per sec: 83885.95022061054, Loss: 2.685608148574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 390 , TFLOPS: 102.6714370706569, Tokens per sec: 83895.47422835164, Loss: 2.708691120147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 391 , TFLOPS: 101.49417439978927, Tokens per sec: 82933.50259454851, Loss: 2.703397035598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 392 , TFLOPS: 102.28395281547373, Tokens per sec: 83578.85086870936, Loss: 2.6869056224823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 393 , TFLOPS: 102.15781705753223, Tokens per sec: 83475.78209387208, Loss: 2.704457998275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 394 , TFLOPS: 100.57489799474328, Tokens per sec: 82182.33817970552, Loss: 2.680175542831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 395 , TFLOPS: 103.24107321787157, Tokens per sec: 84360.9385879797, Loss: 2.7123146057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 396 , TFLOPS: 101.84626852482234, Tokens per sec: 83221.20776782212, Loss: 2.681452751159668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 397 , TFLOPS: 101.71931917434061, Tokens per sec: 83117.47418557649, Loss: 2.7370336055755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 398 , TFLOPS: 102.66527592288594, Tokens per sec: 83890.43979590593, Loss: 2.708894968032837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 399 , TFLOPS: 101.49618834791114, Tokens per sec: 82935.14824340252, Loss: 2.700820207595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 400 , TFLOPS: 102.17540874811638, Tokens per sec: 83490.1567170988, Loss: 2.6777374744415283 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/optimizer.pt +[rank0]:[2024-08-27 00:03:29,006] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007791337004164234, 'preprocessing_with_comm': 0.0015599320031469688, 'state_converting': 3.2734923730022274, : 3.2844709580094786}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400, took 15.78s + eval ppl=11.297107696533203, eval loss=2.424546718597412 +------------------------------------------------------------------ +iteration: 401 , TFLOPS: 102.29485677425835, Tokens per sec: 83587.76077412514, Loss: 2.694633960723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 402 , TFLOPS: 102.0940554405367, Tokens per sec: 83423.680835255, Loss: 2.706956386566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 403 , TFLOPS: 102.26129787218862, Tokens per sec: 83560.33893136098, Loss: 2.677337646484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 404 , TFLOPS: 101.51570368892152, Tokens per sec: 82951.09473091166, Loss: 2.7006263732910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 405 , TFLOPS: 101.88023508015189, Tokens per sec: 83248.96271455882, Loss: 2.689950942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 406 , TFLOPS: 102.23812946484357, Tokens per sec: 83541.40742931147, Loss: 2.6964638233184814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 407 , TFLOPS: 101.82101951235451, Tokens per sec: 83200.57614976726, Loss: 2.6740775108337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 408 , TFLOPS: 103.36594723428091, Tokens per sec: 84462.97636133093, Loss: 2.6754720211029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 409 , TFLOPS: 102.00089323057671, Tokens per sec: 83347.55559528805, Loss: 2.7146668434143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 410 , TFLOPS: 103.24554496356056, Tokens per sec: 84364.59256649515, Loss: 2.679288625717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 411 , TFLOPS: 101.97666391908005, Tokens per sec: 83327.75720114617, Loss: 2.675112009048462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 412 , TFLOPS: 103.3634881994898, Tokens per sec: 84460.96702070191, Loss: 2.7059237957000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 413 , TFLOPS: 101.73985486390774, Tokens per sec: 83134.25442615748, Loss: 2.6987740993499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 414 , TFLOPS: 101.55797473518065, Tokens per sec: 82985.63549096354, Loss: 2.707460880279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 415 , TFLOPS: 102.66192972782352, Tokens per sec: 83887.7055337817, Loss: 2.6720638275146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 416 , TFLOPS: 101.92125852032018, Tokens per sec: 83282.48402355764, Loss: 2.7057628631591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 417 , TFLOPS: 102.62153949455812, Tokens per sec: 83854.70163444345, Loss: 2.6859707832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 418 , TFLOPS: 102.14381165912471, Tokens per sec: 83464.33792230226, Loss: 2.6974942684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 419 , TFLOPS: 101.93980678632528, Tokens per sec: 83297.64028918525, Loss: 2.6574385166168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 420 , TFLOPS: 100.66394676368509, Tokens per sec: 82255.1022211271, Loss: 2.70023250579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 421 , TFLOPS: 102.61619409196183, Tokens per sec: 83850.3337683791, Loss: 2.6613879203796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 422 , TFLOPS: 103.25073057558703, Tokens per sec: 84368.82986357149, Loss: 2.6783061027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 423 , TFLOPS: 102.0805981470569, Tokens per sec: 83412.6845343311, Loss: 2.658961534500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 424 , TFLOPS: 101.58272465287718, Tokens per sec: 83005.85928583294, Loss: 2.6857614517211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 425 , TFLOPS: 102.50027953957394, Tokens per sec: 83755.61700371692, Loss: 2.6602606773376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 426 , TFLOPS: 102.64284900472515, Tokens per sec: 83872.11418375652, Loss: 2.6567468643188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 427 , TFLOPS: 101.50854313082851, Tokens per sec: 82945.2436545647, Loss: 2.6737496852874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 428 , TFLOPS: 101.87317552094866, Tokens per sec: 83243.19416700465, Loss: 2.6442131996154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 429 , TFLOPS: 101.47547994923939, Tokens per sec: 82918.22687776622, Loss: 2.647615909576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 430 , TFLOPS: 102.05866237266163, Tokens per sec: 83394.76024839566, Loss: 2.6458187103271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 431 , TFLOPS: 102.06199631885549, Tokens per sec: 83397.48450165408, Loss: 2.6818459033966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 432 , TFLOPS: 101.85366894545382, Tokens per sec: 83227.25484202389, Loss: 2.682300090789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 433 , TFLOPS: 101.516766413808, Tokens per sec: 82951.96311077339, Loss: 2.666912078857422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 434 , TFLOPS: 101.26150324525752, Tokens per sec: 82743.38100469224, Loss: 2.6688344478607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 435 , TFLOPS: 101.52042437267751, Tokens per sec: 82954.9521231299, Loss: 2.68918776512146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 436 , TFLOPS: 101.65711378775609, Tokens per sec: 83066.64456288918, Loss: 2.6486990451812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 437 , TFLOPS: 101.56268832060456, Tokens per sec: 82989.4870829519, Loss: 2.6430587768554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 438 , TFLOPS: 100.62742094254767, Tokens per sec: 82225.25603241746, Loss: 2.6590166091918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 439 , TFLOPS: 102.76886439823839, Tokens per sec: 83975.084605721, Loss: 2.6509573459625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 440 , TFLOPS: 102.00997911816383, Tokens per sec: 83354.97990792705, Loss: 2.6738393306732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 441 , TFLOPS: 102.25779868882447, Tokens per sec: 83557.47965855713, Loss: 2.6679790019989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 442 , TFLOPS: 102.13157504082108, Tokens per sec: 83454.33906648781, Loss: 2.678312301635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 443 , TFLOPS: 102.80500181613984, Tokens per sec: 84004.61342014818, Loss: 2.6854257583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 444 , TFLOPS: 102.80326939351306, Tokens per sec: 84003.19781302332, Loss: 2.665764331817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 445 , TFLOPS: 102.545572667902, Tokens per sec: 83792.62718482263, Loss: 2.621889352798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 446 , TFLOPS: 101.9618827731555, Tokens per sec: 83315.67914631068, Loss: 2.6513819694519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 447 , TFLOPS: 102.55133012631025, Tokens per sec: 83797.33175229818, Loss: 2.6691205501556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 448 , TFLOPS: 102.74615280739711, Tokens per sec: 83956.52638019623, Loss: 2.6408679485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 449 , TFLOPS: 102.52622676388809, Tokens per sec: 83776.819148645, Loss: 2.6417815685272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 450 , TFLOPS: 103.40153186616192, Tokens per sec: 84492.0534800708, Loss: 2.6768381595611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 451 , TFLOPS: 101.76248830865424, Tokens per sec: 83152.74879649654, Loss: 2.6571474075317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 452 , TFLOPS: 101.3222496670905, Tokens per sec: 82793.01846972399, Loss: 2.643867015838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 453 , TFLOPS: 102.67007999585576, Tokens per sec: 83894.3653275974, Loss: 2.651698112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 454 , TFLOPS: 102.53032449210542, Tokens per sec: 83780.16750785628, Loss: 2.6721441745758057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 455 , TFLOPS: 102.6522853607435, Tokens per sec: 83879.82487317253, Loss: 2.667860984802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 456 , TFLOPS: 101.37588788134367, Tokens per sec: 82836.84763536073, Loss: 2.666029453277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 457 , TFLOPS: 101.94709776204728, Tokens per sec: 83303.5979331341, Loss: 2.6417365074157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 458 , TFLOPS: 101.90627425550929, Tokens per sec: 83270.23999505136, Loss: 2.6392338275909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 459 , TFLOPS: 102.73590363716625, Tokens per sec: 83948.15151936348, Loss: 2.634664297103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 460 , TFLOPS: 102.71746431189436, Tokens per sec: 83933.08427201337, Loss: 2.666874408721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 461 , TFLOPS: 102.03862236996817, Tokens per sec: 83378.38504631926, Loss: 2.6481103897094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 462 , TFLOPS: 101.43953168734053, Tokens per sec: 82888.85262757809, Loss: 2.651137590408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 463 , TFLOPS: 103.35502316648821, Tokens per sec: 84454.05002432699, Loss: 2.636888265609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 464 , TFLOPS: 101.92636831168494, Tokens per sec: 83286.6593656195, Loss: 2.6393001079559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 465 , TFLOPS: 101.34354255605763, Tokens per sec: 82810.41743742676, Loss: 2.6665713787078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 466 , TFLOPS: 102.53199861506425, Tokens per sec: 83781.53547682171, Loss: 2.6343677043914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 467 , TFLOPS: 100.40698589920406, Tokens per sec: 82045.13288399855, Loss: 2.6340770721435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 468 , TFLOPS: 103.2446585818876, Tokens per sec: 84363.86828121268, Loss: 2.639357089996338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 469 , TFLOPS: 100.89077618395167, Tokens per sec: 82440.45037953569, Loss: 2.651721239089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 470 , TFLOPS: 102.65601476856212, Tokens per sec: 83882.87226830462, Loss: 2.6466333866119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 471 , TFLOPS: 101.61164184891467, Tokens per sec: 83029.48827111, Loss: 2.621192216873169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 472 , TFLOPS: 101.32481956174465, Tokens per sec: 82795.11839680087, Loss: 2.653989791870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 473 , TFLOPS: 101.53010839641293, Tokens per sec: 82962.86518821307, Loss: 2.6233839988708496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 474 , TFLOPS: 101.16144795990381, Tokens per sec: 82661.62325537755, Loss: 2.6287946701049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 475 , TFLOPS: 102.14609290155508, Tokens per sec: 83466.20198421653, Loss: 2.6372735500335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 476 , TFLOPS: 100.49314335025565, Tokens per sec: 82115.53435514291, Loss: 2.6307852268218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 477 , TFLOPS: 102.72213791130703, Tokens per sec: 83936.9031904026, Loss: 2.6461782455444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 478 , TFLOPS: 102.13247876789153, Tokens per sec: 83455.07752514107, Loss: 2.6482789516448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 479 , TFLOPS: 102.14666894513421, Tokens per sec: 83466.67268425373, Loss: 2.634648084640503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 480 , TFLOPS: 102.64165726054517, Tokens per sec: 83871.14037890884, Loss: 2.66127347946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 481 , TFLOPS: 102.85804396089235, Tokens per sec: 84047.95552205172, Loss: 2.631625175476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 482 , TFLOPS: 103.38359300969319, Tokens per sec: 84477.39517866296, Loss: 2.623292922973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 483 , TFLOPS: 102.68070552571317, Tokens per sec: 83903.04772156937, Loss: 2.6394219398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 484 , TFLOPS: 102.09363609378786, Tokens per sec: 83423.33817623158, Loss: 2.632636070251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 485 , TFLOPS: 102.54218626871497, Tokens per sec: 83789.86007087324, Loss: 2.6428873538970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 486 , TFLOPS: 102.75298587115515, Tokens per sec: 83962.10985249163, Loss: 2.6272268295288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 487 , TFLOPS: 102.65670075214638, Tokens per sec: 83883.4328031501, Loss: 2.6211860179901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 488 , TFLOPS: 102.60322079894533, Tokens per sec: 83839.73295669306, Loss: 2.635227918624878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 489 , TFLOPS: 101.95657657972683, Tokens per sec: 83311.34331906654, Loss: 2.652369499206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 490 , TFLOPS: 102.14762325051375, Tokens per sec: 83467.45247174536, Loss: 2.6495521068573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 491 , TFLOPS: 102.66974641651342, Tokens per sec: 83894.09275132968, Loss: 2.6254241466522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 492 , TFLOPS: 102.59260004346234, Tokens per sec: 83831.05446398533, Loss: 2.6273410320281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 493 , TFLOPS: 102.7170341048823, Tokens per sec: 83932.73273878929, Loss: 2.620610237121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 494 , TFLOPS: 100.32452864830269, Tokens per sec: 81977.75494164857, Loss: 2.6012377738952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 495 , TFLOPS: 102.77485672068372, Tokens per sec: 83979.98108664718, Loss: 2.616610050201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 496 , TFLOPS: 102.07201337673442, Tokens per sec: 83405.66969750896, Loss: 2.634077310562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 497 , TFLOPS: 102.85488200133003, Tokens per sec: 84045.37180349727, Loss: 2.6340558528900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 498 , TFLOPS: 102.6508962478724, Tokens per sec: 83878.6897932866, Loss: 2.62162709236145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 499 , TFLOPS: 101.9490206835103, Tokens per sec: 83305.16920176194, Loss: 2.623035192489624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 500 , TFLOPS: 101.64772276162607, Tokens per sec: 83058.97091369182, Loss: 2.616741180419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 501 , TFLOPS: 103.45549493551023, Tokens per sec: 84536.1480931683, Loss: 2.599799633026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 502 , TFLOPS: 100.61481050702835, Tokens per sec: 82214.95172093312, Loss: 2.611288547515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 503 , TFLOPS: 101.53502169411279, Tokens per sec: 82966.87997024314, Loss: 2.6255199909210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 504 , TFLOPS: 102.55704404673968, Tokens per sec: 83802.00074377046, Loss: 2.628739595413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 505 , TFLOPS: 100.92695419353457, Tokens per sec: 82470.01236247068, Loss: 2.6239097118377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 506 , TFLOPS: 102.63958807777237, Tokens per sec: 83869.44959639967, Loss: 2.6222591400146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 507 , TFLOPS: 100.98996480106719, Tokens per sec: 82521.49995192293, Loss: 2.627751111984253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 508 , TFLOPS: 101.04054185890743, Tokens per sec: 82562.82776785348, Loss: 2.6102185249328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 509 , TFLOPS: 101.55279028348627, Tokens per sec: 82981.3991420244, Loss: 2.6184051036834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 510 , TFLOPS: 102.64301949610604, Tokens per sec: 83872.25349665267, Loss: 2.6115822792053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 511 , TFLOPS: 100.94446668773175, Tokens per sec: 82484.3222722909, Loss: 2.6204676628112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 512 , TFLOPS: 100.93350744320902, Tokens per sec: 82475.36719147534, Loss: 2.590601921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 513 , TFLOPS: 102.05162646568087, Tokens per sec: 83389.01102768132, Loss: 2.6289491653442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 514 , TFLOPS: 100.48062902990598, Tokens per sec: 82105.30858183793, Loss: 2.6178393363952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 515 , TFLOPS: 102.43948708981327, Tokens per sec: 83705.94192808062, Loss: 2.5985946655273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 516 , TFLOPS: 102.0667375117231, Tokens per sec: 83401.35865239536, Loss: 2.618314743041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 517 , TFLOPS: 101.48091298292717, Tokens per sec: 82922.66634945123, Loss: 2.6300296783447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 518 , TFLOPS: 103.4046874294227, Tokens per sec: 84494.63197204261, Loss: 2.620435953140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 519 , TFLOPS: 101.48620936437847, Tokens per sec: 82926.99415907609, Loss: 2.602114200592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 520 , TFLOPS: 102.75510675215928, Tokens per sec: 83963.84287895644, Loss: 2.6096243858337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 521 , TFLOPS: 102.76380497990931, Tokens per sec: 83970.9504247633, Loss: 2.6317121982574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 522 , TFLOPS: 102.10493676371246, Tokens per sec: 83432.5722445321, Loss: 2.630302906036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 523 , TFLOPS: 102.66686779492049, Tokens per sec: 83891.7405555238, Loss: 2.6182854175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 524 , TFLOPS: 102.71706778849746, Tokens per sec: 83932.76026253833, Loss: 2.5953822135925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 525 , TFLOPS: 102.54335738672953, Tokens per sec: 83790.8170215502, Loss: 2.613003969192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 526 , TFLOPS: 102.21649824666221, Tokens per sec: 83523.7320040984, Loss: 2.5985536575317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 527 , TFLOPS: 102.71419402311726, Tokens per sec: 83930.41203486887, Loss: 2.6018505096435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 528 , TFLOPS: 102.03730960230999, Tokens per sec: 83377.31235007213, Loss: 2.628756046295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 529 , TFLOPS: 102.57462844387919, Tokens per sec: 83816.3694073358, Loss: 2.610095262527466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 530 , TFLOPS: 102.0415378714091, Tokens per sec: 83380.76737759834, Loss: 2.6023952960968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 531 , TFLOPS: 103.32737415183354, Tokens per sec: 84431.4573027037, Loss: 2.601259231567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 532 , TFLOPS: 102.80488340564769, Tokens per sec: 84004.51666388687, Loss: 2.580073833465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 533 , TFLOPS: 102.14739510243332, Tokens per sec: 83467.26604607578, Loss: 2.6174156665802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 534 , TFLOPS: 102.57732641815554, Tokens per sec: 83818.57399156905, Loss: 2.6148457527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 535 , TFLOPS: 102.08975764068208, Tokens per sec: 83420.16899235858, Loss: 2.596099853515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 536 , TFLOPS: 101.33951184043534, Tokens per sec: 82807.12383593192, Loss: 2.6062815189361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 537 , TFLOPS: 102.09283409414407, Tokens per sec: 83422.68284167732, Loss: 2.619462728500366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 538 , TFLOPS: 102.14266372367273, Tokens per sec: 83463.39991469398, Loss: 2.5963990688323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 539 , TFLOPS: 102.81265665793775, Tokens per sec: 84010.86838843439, Loss: 2.5871317386627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 540 , TFLOPS: 101.47238407916646, Tokens per sec: 82915.69716263485, Loss: 2.6025190353393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 541 , TFLOPS: 101.48175961504452, Tokens per sec: 82923.35815434858, Loss: 2.594595193862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 542 , TFLOPS: 102.54889583808881, Tokens per sec: 83795.3426327283, Loss: 2.5918173789978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 543 , TFLOPS: 100.37184726316178, Tokens per sec: 82016.42019993946, Loss: 2.6119117736816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 544 , TFLOPS: 102.13341073253325, Tokens per sec: 83455.83905744072, Loss: 2.613637924194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 545 , TFLOPS: 102.13154639153935, Tokens per sec: 83454.3156564222, Loss: 2.6062254905700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 546 , TFLOPS: 100.56314446321137, Tokens per sec: 82172.73406652798, Loss: 2.5966057777404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 547 , TFLOPS: 101.30542982656355, Tokens per sec: 82779.2745450482, Loss: 2.5979020595550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 548 , TFLOPS: 102.84795944059543, Tokens per sec: 84039.71520091846, Loss: 2.5733160972595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 549 , TFLOPS: 101.54969323344616, Tokens per sec: 82978.86846271147, Loss: 2.5946712493896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 550 , TFLOPS: 101.00922763724955, Tokens per sec: 82537.24011123706, Loss: 2.5853657722473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 551 , TFLOPS: 102.00090147930518, Tokens per sec: 83347.56233553644, Loss: 2.57668399810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 552 , TFLOPS: 100.22407820448556, Tokens per sec: 81895.67429818114, Loss: 2.5978901386260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 553 , TFLOPS: 102.77607540769269, Tokens per sec: 83980.976907174, Loss: 2.5728769302368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 554 , TFLOPS: 101.59099001630906, Tokens per sec: 83012.61312706252, Loss: 2.5880870819091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 555 , TFLOPS: 100.84412703951067, Tokens per sec: 82402.33216275713, Loss: 2.579909563064575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 556 , TFLOPS: 102.45858939004894, Tokens per sec: 83721.55090934013, Loss: 2.60117769241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 557 , TFLOPS: 101.32460481279425, Tokens per sec: 82794.94291990537, Loss: 2.5969417095184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 558 , TFLOPS: 101.6621275575337, Tokens per sec: 83070.74144324037, Loss: 2.593202590942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 559 , TFLOPS: 102.02077766993342, Tokens per sec: 83363.80368255744, Loss: 2.5725810527801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 560 , TFLOPS: 102.0684547633099, Tokens per sec: 83402.76186287293, Loss: 2.5822834968566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 561 , TFLOPS: 102.72851645848682, Tokens per sec: 83942.11526550163, Loss: 2.5927042961120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 562 , TFLOPS: 101.01169027082463, Tokens per sec: 82539.25239252507, Loss: 2.597472906112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 563 , TFLOPS: 102.66959131019696, Tokens per sec: 83893.96600996572, Loss: 2.592088460922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 564 , TFLOPS: 102.57518966527239, Tokens per sec: 83816.82799578333, Loss: 2.6080586910247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 565 , TFLOPS: 102.0553223067229, Tokens per sec: 83392.03099453617, Loss: 2.5839240550994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 566 , TFLOPS: 101.39344746381552, Tokens per sec: 82851.19602222234, Loss: 2.595102310180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 567 , TFLOPS: 102.65162436157577, Tokens per sec: 83879.28475373694, Loss: 2.6143293380737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 568 , TFLOPS: 102.76005137360085, Tokens per sec: 83967.88325642222, Loss: 2.5709919929504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 569 , TFLOPS: 102.15111164808808, Tokens per sec: 83470.30293120268, Loss: 2.5927929878234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 570 , TFLOPS: 102.71225751047245, Tokens per sec: 83928.82966053653, Loss: 2.5960047245025635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 571 , TFLOPS: 101.34475326873617, Tokens per sec: 82811.40674192291, Loss: 2.6015055179595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 572 , TFLOPS: 102.72350125215833, Tokens per sec: 83938.01721130771, Loss: 2.5719308853149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 573 , TFLOPS: 102.80140309331038, Tokens per sec: 84001.67281108486, Loss: 2.5705060958862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 574 , TFLOPS: 100.70202878540557, Tokens per sec: 82286.22002139338, Loss: 2.5822670459747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 575 , TFLOPS: 101.54756222033889, Tokens per sec: 82977.12715704436, Loss: 2.5663604736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 576 , TFLOPS: 102.28433876285166, Tokens per sec: 83579.16623624704, Loss: 2.5765113830566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 577 , TFLOPS: 102.03451551692521, Tokens per sec: 83375.02923097806, Loss: 2.567470073699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 578 , TFLOPS: 101.06852740264766, Tokens per sec: 82585.69547605561, Loss: 2.5633721351623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 579 , TFLOPS: 102.11881135518912, Tokens per sec: 83443.90953039144, Loss: 2.583566904067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 580 , TFLOPS: 102.5356670939966, Tokens per sec: 83784.53308539238, Loss: 2.5662052631378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 581 , TFLOPS: 100.39780089205348, Tokens per sec: 82037.62757821269, Loss: 2.573852062225342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 582 , TFLOPS: 102.15314194485137, Tokens per sec: 83471.96193895246, Loss: 2.5730018615722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 583 , TFLOPS: 102.7061550970909, Tokens per sec: 83923.84322147237, Loss: 2.5891456604003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 584 , TFLOPS: 100.41668586360345, Tokens per sec: 82053.05897460859, Loss: 2.579669237136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 585 , TFLOPS: 101.45408585545974, Tokens per sec: 82900.7452130059, Loss: 2.59645938873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 586 , TFLOPS: 102.00446288795742, Tokens per sec: 83350.47245422499, Loss: 2.5822627544403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 587 , TFLOPS: 101.51216319635341, Tokens per sec: 82948.20170329389, Loss: 2.574944257736206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 588 , TFLOPS: 101.85112919710477, Tokens per sec: 83225.17954826893, Loss: 2.580817699432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 589 , TFLOPS: 101.55603152259656, Tokens per sec: 82984.04764193833, Loss: 2.5798094272613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 590 , TFLOPS: 100.86763929187181, Tokens per sec: 82421.54462942063, Loss: 2.584151268005371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 591 , TFLOPS: 101.4041254609545, Tokens per sec: 82859.9212885607, Loss: 2.5629703998565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 592 , TFLOPS: 102.70148678174846, Tokens per sec: 83920.02862083293, Loss: 2.586181640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 593 , TFLOPS: 100.34562442970316, Tokens per sec: 81994.99284768463, Loss: 2.567603588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 594 , TFLOPS: 101.58126834805273, Tokens per sec: 83004.66930167226, Loss: 2.5816564559936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 595 , TFLOPS: 101.6277269606877, Tokens per sec: 83042.63183000738, Loss: 2.594780445098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 596 , TFLOPS: 101.42832629218086, Tokens per sec: 82879.69641074058, Loss: 2.5697712898254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 597 , TFLOPS: 101.58460356891096, Tokens per sec: 83007.39459649175, Loss: 2.5924899578094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 598 , TFLOPS: 101.02715489655914, Tokens per sec: 82551.88893630763, Loss: 2.5870418548583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 599 , TFLOPS: 102.16648388955618, Tokens per sec: 83482.86398542297, Loss: 2.5648305416107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 600 , TFLOPS: 102.66173542476832, Tokens per sec: 83887.5467637535, Loss: 2.570026397705078 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/optimizer.pt +[rank0]:[2024-08-27 00:56:17,097] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007729057004326023, 'preprocessing_with_comm': 0.001662016991758719, 'state_converting': 3.346408262004843, : 3.3574239049921744}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600, took 15.64s + eval ppl=9.844626426696777, eval loss=2.286925792694092 +------------------------------------------------------------------ +iteration: 601 , TFLOPS: 100.66187252832218, Tokens per sec: 82253.40731001651, Loss: 2.5616564750671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 602 , TFLOPS: 102.11896360027146, Tokens per sec: 83444.03393376735, Loss: 2.584953546524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 603 , TFLOPS: 100.10240255656758, Tokens per sec: 81796.24999405752, Loss: 2.569500207901001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 604 , TFLOPS: 102.64971320904183, Tokens per sec: 83877.72310180402, Loss: 2.5810184478759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 605 , TFLOPS: 101.1185701687871, Tokens per sec: 82626.58670848355, Loss: 2.55959153175354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 606 , TFLOPS: 101.52233320902364, Tokens per sec: 82956.51188244614, Loss: 2.5582315921783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 607 , TFLOPS: 102.13487528457316, Tokens per sec: 83457.03578061346, Loss: 2.578044891357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 608 , TFLOPS: 101.72391250727362, Tokens per sec: 83121.22751615896, Loss: 2.5815136432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 609 , TFLOPS: 102.10298928940033, Tokens per sec: 83430.9809131392, Loss: 2.5762779712677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 610 , TFLOPS: 103.3226906431024, Tokens per sec: 84427.63028715525, Loss: 2.5717616081237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 611 , TFLOPS: 100.68076573369136, Tokens per sec: 82268.84543447805, Loss: 2.570974588394165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 612 , TFLOPS: 102.63198833528484, Tokens per sec: 83863.23965117834, Loss: 2.563427686691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 613 , TFLOPS: 102.62005424943936, Tokens per sec: 83853.4880024233, Loss: 2.569070816040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 614 , TFLOPS: 102.02137869723039, Tokens per sec: 83364.2947974336, Loss: 2.5794684886932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 615 , TFLOPS: 101.11472018897081, Tokens per sec: 82623.44079086836, Loss: 2.563856363296509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 616 , TFLOPS: 101.98714928941789, Tokens per sec: 83336.32506716666, Loss: 2.5731639862060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 617 , TFLOPS: 102.08161845551797, Tokens per sec: 83413.51825463973, Loss: 2.5688228607177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 618 , TFLOPS: 102.66354393280211, Tokens per sec: 83889.02454222304, Loss: 2.5651936531066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 619 , TFLOPS: 102.0703982925663, Tokens per sec: 83404.34997065921, Loss: 2.5660393238067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 620 , TFLOPS: 102.00247055346814, Tokens per sec: 83348.84446642654, Loss: 2.5672593116760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 621 , TFLOPS: 101.95417388907329, Tokens per sec: 83309.38001868277, Loss: 2.5836329460144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 622 , TFLOPS: 100.66960745502816, Tokens per sec: 82259.72771774269, Loss: 2.5801117420196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 623 , TFLOPS: 103.31665493786275, Tokens per sec: 84422.69835703105, Loss: 2.568633556365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 624 , TFLOPS: 102.11538220977714, Tokens per sec: 83441.10748740124, Loss: 2.556692361831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 625 , TFLOPS: 102.75486036967632, Tokens per sec: 83963.64155348716, Loss: 2.559199333190918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 626 , TFLOPS: 101.83169959867065, Tokens per sec: 83209.30312322607, Loss: 2.5698416233062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 627 , TFLOPS: 102.0801610311988, Tokens per sec: 83412.3273557111, Loss: 2.571089267730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 628 , TFLOPS: 101.63286015828868, Tokens per sec: 83046.82629790739, Loss: 2.5541677474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 629 , TFLOPS: 102.7184625896986, Tokens per sec: 83933.89999050516, Loss: 2.5366086959838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 630 , TFLOPS: 101.41305339786462, Tokens per sec: 82867.2165356355, Loss: 2.544672727584839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 631 , TFLOPS: 102.11149686369417, Tokens per sec: 83437.93267109914, Loss: 2.5493743419647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 632 , TFLOPS: 102.13184015452485, Tokens per sec: 83454.55569771978, Loss: 2.5505177974700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 633 , TFLOPS: 101.67388431894652, Tokens per sec: 83080.34819563676, Loss: 2.5638318061828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 634 , TFLOPS: 102.81908081495922, Tokens per sec: 84016.11773250916, Loss: 2.561347484588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 635 , TFLOPS: 102.63313653009419, Tokens per sec: 83864.17787071422, Loss: 2.5582218170166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 636 , TFLOPS: 100.33353165400648, Tokens per sec: 81985.11152936712, Loss: 2.580814838409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 637 , TFLOPS: 102.77775118407085, Tokens per sec: 83982.34622719092, Loss: 2.57840895652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 638 , TFLOPS: 102.22696604403667, Tokens per sec: 83532.28551079814, Loss: 2.5716285705566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 639 , TFLOPS: 101.39909809579135, Tokens per sec: 82855.81329907, Loss: 2.580416440963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 640 , TFLOPS: 102.03289539462503, Tokens per sec: 83373.70538734092, Loss: 2.5528323650360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 641 , TFLOPS: 101.11047770926885, Tokens per sec: 82619.97415149276, Loss: 2.567647933959961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 642 , TFLOPS: 102.65774519848213, Tokens per sec: 83884.28624713748, Loss: 2.574845552444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 643 , TFLOPS: 100.31519994761673, Tokens per sec: 81970.13222017596, Loss: 2.5504910945892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 644 , TFLOPS: 101.00964403260215, Tokens per sec: 82537.58035859863, Loss: 2.5668416023254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 645 , TFLOPS: 102.00773310133157, Tokens per sec: 83353.1446297558, Loss: 2.5645623207092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 646 , TFLOPS: 101.72327863863366, Tokens per sec: 83120.70956577604, Loss: 2.5536181926727295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 647 , TFLOPS: 101.35470949358313, Tokens per sec: 82819.54222954139, Loss: 2.5444772243499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 648 , TFLOPS: 103.2881223397696, Tokens per sec: 84399.38363663472, Loss: 2.5657243728637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 649 , TFLOPS: 102.26137959673493, Tokens per sec: 83560.40571059154, Loss: 2.565603494644165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 650 , TFLOPS: 101.5633280144533, Tokens per sec: 82990.00979326289, Loss: 2.572841167449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 651 , TFLOPS: 103.48998541380783, Tokens per sec: 84564.33115084918, Loss: 2.5484519004821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 652 , TFLOPS: 102.12123017123845, Tokens per sec: 83445.8860072512, Loss: 2.550144910812378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 653 , TFLOPS: 101.42093084635138, Tokens per sec: 82873.65340157761, Loss: 2.5473670959472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 654 , TFLOPS: 102.0871022329068, Tokens per sec: 83417.99919030913, Loss: 2.5558090209960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 655 , TFLOPS: 102.09457907379937, Tokens per sec: 83424.10870947344, Loss: 2.5681815147399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 656 , TFLOPS: 102.64223159735484, Tokens per sec: 83871.60968430074, Loss: 2.542442560195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 657 , TFLOPS: 101.93505134995975, Tokens per sec: 83293.75449971514, Loss: 2.5645556449890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 658 , TFLOPS: 101.92516293110566, Tokens per sec: 83285.67441811893, Loss: 2.5314066410064697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 659 , TFLOPS: 102.08030172589615, Tokens per sec: 83412.44232097009, Loss: 2.5736541748046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 660 , TFLOPS: 99.6129706526903, Tokens per sec: 81396.32258630134, Loss: 2.5633912086486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 661 , TFLOPS: 103.33742828326922, Tokens per sec: 84439.67279231617, Loss: 2.5417516231536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 662 , TFLOPS: 101.4540543028507, Tokens per sec: 82900.71943055681, Loss: 2.5616228580474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 663 , TFLOPS: 102.63486630030415, Tokens per sec: 83865.59131048102, Loss: 2.54408597946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 664 , TFLOPS: 101.78987280025468, Tokens per sec: 83175.12536952281, Loss: 2.559842586517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 665 , TFLOPS: 101.49617950036965, Tokens per sec: 82935.14101384861, Loss: 2.533412456512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 666 , TFLOPS: 102.14343528376982, Tokens per sec: 83464.03037631104, Loss: 2.5429232120513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 667 , TFLOPS: 102.12270232425668, Tokens per sec: 83447.08894137898, Loss: 2.558215379714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 668 , TFLOPS: 102.84951987866798, Tokens per sec: 84040.99027503679, Loss: 2.540024757385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 669 , TFLOPS: 102.71630262511535, Tokens per sec: 83932.13502784174, Loss: 2.534769296646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 670 , TFLOPS: 102.0867151308991, Tokens per sec: 83417.68287929373, Loss: 2.5647401809692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 671 , TFLOPS: 101.29098075611881, Tokens per sec: 82767.46783763566, Loss: 2.557149887084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 672 , TFLOPS: 103.3135421781336, Tokens per sec: 84420.1548409268, Loss: 2.5478999614715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 673 , TFLOPS: 102.66424333605536, Tokens per sec: 83889.5960426255, Loss: 2.5307655334472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 674 , TFLOPS: 100.91600355887176, Tokens per sec: 82461.0643169933, Loss: 2.550854444503784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 675 , TFLOPS: 102.06536968353826, Tokens per sec: 83400.2409647745, Loss: 2.56469988822937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 676 , TFLOPS: 102.12994172012199, Tokens per sec: 83453.00443809916, Loss: 2.5400381088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 677 , TFLOPS: 100.62548919439752, Tokens per sec: 82223.67755127633, Loss: 2.56591534614563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 678 , TFLOPS: 102.6527036520321, Tokens per sec: 83880.16666975222, Loss: 2.5341591835021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 679 , TFLOPS: 100.49624325980801, Tokens per sec: 82118.0673710369, Loss: 2.546137809753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 680 , TFLOPS: 102.60820619551002, Tokens per sec: 83843.80665256159, Loss: 2.542839288711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 681 , TFLOPS: 100.43379874662135, Tokens per sec: 82067.04235184733, Loss: 2.548417568206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 682 , TFLOPS: 101.56997766047986, Tokens per sec: 82995.44339020831, Loss: 2.5485455989837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 683 , TFLOPS: 102.25025685485672, Tokens per sec: 83551.317031878, Loss: 2.5335512161254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 684 , TFLOPS: 101.68710459291889, Tokens per sec: 83091.15082181971, Loss: 2.533641815185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 685 , TFLOPS: 102.1808215509321, Tokens per sec: 83494.57965761746, Loss: 2.5484540462493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 686 , TFLOPS: 102.55242286524823, Tokens per sec: 83798.22465741402, Loss: 2.536715030670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 687 , TFLOPS: 102.21635303120881, Tokens per sec: 83523.61334481325, Loss: 2.531450033187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 688 , TFLOPS: 102.30981954382278, Tokens per sec: 83599.9872383122, Loss: 2.5347483158111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 689 , TFLOPS: 102.76542267310053, Tokens per sec: 83972.27228351297, Loss: 2.5503668785095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 690 , TFLOPS: 102.16821687724007, Tokens per sec: 83484.28005427054, Loss: 2.565098762512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 691 , TFLOPS: 102.0182239092373, Tokens per sec: 83361.716938953, Loss: 2.538743495941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 692 , TFLOPS: 101.28853123917943, Tokens per sec: 82765.46627428831, Loss: 2.5307986736297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 693 , TFLOPS: 102.65850947660199, Tokens per sec: 83884.91075846349, Loss: 2.5552194118499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 694 , TFLOPS: 101.8956896476305, Tokens per sec: 83261.59103948156, Loss: 2.542059898376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 695 , TFLOPS: 102.84466571901989, Tokens per sec: 84037.02381623116, Loss: 2.5069973468780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 696 , TFLOPS: 101.45450063150176, Tokens per sec: 82901.08413718712, Loss: 2.548326253890991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 697 , TFLOPS: 102.70435463304305, Tokens per sec: 83922.37201595081, Loss: 2.552000045776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 698 , TFLOPS: 99.57069056457449, Tokens per sec: 81361.77443791562, Loss: 2.536149024963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 699 , TFLOPS: 103.26498809110663, Tokens per sec: 84380.48004652366, Loss: 2.564690351486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 700 , TFLOPS: 101.88926738844181, Tokens per sec: 83256.34324617482, Loss: 2.5374536514282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 701 , TFLOPS: 102.76346739090833, Tokens per sec: 83970.67457210028, Loss: 2.528555393218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 702 , TFLOPS: 101.25990357380942, Tokens per sec: 82742.07387197281, Loss: 2.5457165241241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 703 , TFLOPS: 100.88786372950831, Tokens per sec: 82438.07053803677, Loss: 2.54884672164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 704 , TFLOPS: 102.23936695221566, Tokens per sec: 83542.41861209896, Loss: 2.5428452491760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 705 , TFLOPS: 102.16408640940652, Tokens per sec: 83480.90494267606, Loss: 2.5475363731384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 706 , TFLOPS: 102.73350160865353, Tokens per sec: 83946.18876003203, Loss: 2.564511299133301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 707 , TFLOPS: 102.67909943525171, Tokens per sec: 83901.73534370888, Loss: 2.513201951980591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 708 , TFLOPS: 102.11011191293922, Tokens per sec: 83436.80099218547, Loss: 2.531266212463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 709 , TFLOPS: 102.1728850021091, Tokens per sec: 83488.09449926925, Loss: 2.5520927906036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 710 , TFLOPS: 103.32180388710121, Tokens per sec: 84426.90569599955, Loss: 2.535342216491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 711 , TFLOPS: 102.70939026903022, Tokens per sec: 83926.48676374472, Loss: 2.544931411743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 712 , TFLOPS: 100.88504662074403, Tokens per sec: 82435.76860594665, Loss: 2.5304336547851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 713 , TFLOPS: 101.43939766897587, Tokens per sec: 82888.74311772238, Loss: 2.5455565452575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 714 , TFLOPS: 102.19782516712046, Tokens per sec: 83508.473750117, Loss: 2.552314519882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 715 , TFLOPS: 100.97653277531397, Tokens per sec: 82510.52429790888, Loss: 2.5215022563934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 716 , TFLOPS: 102.78294623813358, Tokens per sec: 83986.59123959817, Loss: 2.543182134628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 717 , TFLOPS: 100.4291734058725, Tokens per sec: 82063.26286685457, Loss: 2.515648126602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 718 , TFLOPS: 102.52489913523505, Tokens per sec: 83775.7343090964, Loss: 2.540332317352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 719 , TFLOPS: 100.47319099810325, Tokens per sec: 82099.23077457993, Loss: 2.5114824771881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 720 , TFLOPS: 102.28646453591955, Tokens per sec: 83580.90326014308, Loss: 2.5179250240325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 721 , TFLOPS: 101.56173482098649, Tokens per sec: 82988.70795386871, Loss: 2.531561851501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 722 , TFLOPS: 101.63818669560757, Tokens per sec: 83051.17874866794, Loss: 2.5301401615142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 723 , TFLOPS: 102.0918056226029, Tokens per sec: 83421.84245110583, Loss: 2.5314955711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 724 , TFLOPS: 102.6643074044006, Tokens per sec: 83889.64839451968, Loss: 2.5477938652038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 725 , TFLOPS: 102.1961665702196, Tokens per sec: 83507.11846789322, Loss: 2.5066182613372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 726 , TFLOPS: 102.20608450363608, Tokens per sec: 83515.22266659814, Loss: 2.511563777923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 727 , TFLOPS: 102.67619845105607, Tokens per sec: 83899.36487484463, Loss: 2.5405492782592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 728 , TFLOPS: 101.50505871506621, Tokens per sec: 82942.39644874849, Loss: 2.5415916442871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 729 , TFLOPS: 102.64938542545691, Tokens per sec: 83877.45526139888, Loss: 2.5247926712036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 730 , TFLOPS: 100.69531284324437, Tokens per sec: 82280.73225216972, Loss: 2.520866870880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 731 , TFLOPS: 102.52695493008922, Tokens per sec: 83777.41415199259, Loss: 2.5314841270446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 732 , TFLOPS: 102.66975745896423, Tokens per sec: 83894.10177440051, Loss: 2.5396370887756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 733 , TFLOPS: 101.9621213530294, Tokens per sec: 83315.87409606724, Loss: 2.5201351642608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 734 , TFLOPS: 102.17390950996237, Tokens per sec: 83488.93165100933, Loss: 2.5200867652893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 735 , TFLOPS: 102.08135966418646, Tokens per sec: 83413.30678958086, Loss: 2.5306005477905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 736 , TFLOPS: 99.85313440456919, Tokens per sec: 81592.56657032655, Loss: 2.5487117767333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 737 , TFLOPS: 102.8212772099569, Tokens per sec: 84017.912463402, Loss: 2.52882719039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 738 , TFLOPS: 101.87883986621034, Tokens per sec: 83247.82264933118, Loss: 2.5197601318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 739 , TFLOPS: 102.79006716295919, Tokens per sec: 83992.40993058209, Loss: 2.527991771697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 740 , TFLOPS: 101.62457342477339, Tokens per sec: 83040.05499463255, Loss: 2.524503707885742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 741 , TFLOPS: 102.11022324116826, Tokens per sec: 83436.89196134725, Loss: 2.5188093185424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 742 , TFLOPS: 101.06718736215018, Tokens per sec: 82584.6004944694, Loss: 2.515900135040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 743 , TFLOPS: 102.24486515233185, Tokens per sec: 83546.91133295017, Loss: 2.5345451831817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 744 , TFLOPS: 102.71388654514317, Tokens per sec: 83930.16078670105, Loss: 2.508561611175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 745 , TFLOPS: 101.53676351062384, Tokens per sec: 82968.30325335267, Loss: 2.5325005054473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 746 , TFLOPS: 102.12483015951362, Tokens per sec: 83448.82765034278, Loss: 2.5242998600006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 747 , TFLOPS: 102.10482681705847, Tokens per sec: 83432.48240429083, Loss: 2.5249478816986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 748 , TFLOPS: 103.3077049416127, Tokens per sec: 84415.38508470173, Loss: 2.525418519973755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 749 , TFLOPS: 102.63360312095594, Tokens per sec: 83864.55913411848, Loss: 2.5450878143310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 750 , TFLOPS: 101.00826060267569, Tokens per sec: 82536.44992239306, Loss: 2.5259323120117188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 751 , TFLOPS: 102.06205953397902, Tokens per sec: 83397.53615635881, Loss: 2.5275070667266846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 752 , TFLOPS: 102.28888439990024, Tokens per sec: 83582.88059329454, Loss: 2.5091938972473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 753 , TFLOPS: 101.40099073010222, Tokens per sec: 82857.35981928615, Loss: 2.5264663696289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 754 , TFLOPS: 99.46161104885837, Tokens per sec: 81272.64275766745, Loss: 2.5153963565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 755 , TFLOPS: 100.50820129820166, Tokens per sec: 82127.83859203565, Loss: 2.5179271697998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 756 , TFLOPS: 102.59821806791533, Tokens per sec: 83835.6450963867, Loss: 2.540914535522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 757 , TFLOPS: 101.12801022337008, Tokens per sec: 82634.30042009192, Loss: 2.5096852779388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 758 , TFLOPS: 102.19932357843378, Tokens per sec: 83509.69814057364, Loss: 2.5178732872009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 759 , TFLOPS: 101.65619346217862, Tokens per sec: 83065.89254116855, Loss: 2.512028694152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 760 , TFLOPS: 100.7805183786109, Tokens per sec: 82350.35588850334, Loss: 2.5212650299072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 761 , TFLOPS: 102.765645848729, Tokens per sec: 83972.45464606404, Loss: 2.5220067501068115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 762 , TFLOPS: 102.72542995388079, Tokens per sec: 83939.59320313434, Loss: 2.502967596054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 763 , TFLOPS: 102.30046376478872, Tokens per sec: 83592.34239042428, Loss: 2.5364160537719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 764 , TFLOPS: 102.2230570579817, Tokens per sec: 83529.09137766638, Loss: 2.5085654258728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 765 , TFLOPS: 102.6458338112995, Tokens per sec: 83874.55314604417, Loss: 2.5257740020751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 766 , TFLOPS: 101.58823679401773, Tokens per sec: 83010.36339825402, Loss: 2.5152204036712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 767 , TFLOPS: 102.67233706907236, Tokens per sec: 83896.20964022493, Loss: 2.518888235092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 768 , TFLOPS: 101.40472165248201, Tokens per sec: 82860.40845200518, Loss: 2.5148816108703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 769 , TFLOPS: 102.04360373156297, Tokens per sec: 83382.45544511004, Loss: 2.509267568588257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 770 , TFLOPS: 102.65523265603296, Tokens per sec: 83882.23318402367, Loss: 2.513357400894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 771 , TFLOPS: 102.84340123031285, Tokens per sec: 84035.9905699578, Loss: 2.5192782878875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 772 , TFLOPS: 101.53036688484815, Tokens per sec: 82963.07640576756, Loss: 2.531768798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 773 , TFLOPS: 102.05635154838248, Tokens per sec: 83392.87201439125, Loss: 2.521836757659912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 774 , TFLOPS: 100.81592243499229, Tokens per sec: 82379.28545435405, Loss: 2.4953672885894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 775 , TFLOPS: 101.90123517276905, Tokens per sec: 83266.12243082678, Loss: 2.5174002647399902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 776 , TFLOPS: 102.77274485215699, Tokens per sec: 83978.25542450967, Loss: 2.5323596000671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 777 , TFLOPS: 102.10766115660213, Tokens per sec: 83434.79841609435, Loss: 2.511509656906128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 778 , TFLOPS: 102.23254618353204, Tokens per sec: 83536.84518642537, Loss: 2.528083086013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 779 , TFLOPS: 101.431712952952, Tokens per sec: 82882.46373843709, Loss: 2.5157477855682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 780 , TFLOPS: 102.2627826050652, Tokens per sec: 83561.55214481485, Loss: 2.5276060104370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 781 , TFLOPS: 102.14321419162108, Tokens per sec: 83463.84971622488, Loss: 2.5257983207702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 782 , TFLOPS: 102.83858635151084, Tokens per sec: 84032.05620854266, Loss: 2.516998529434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 783 , TFLOPS: 101.22320667325445, Tokens per sec: 82712.08788986708, Loss: 2.5234382152557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 784 , TFLOPS: 102.28419233812467, Tokens per sec: 83579.04658883331, Loss: 2.5271670818328857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 785 , TFLOPS: 101.56894348571892, Tokens per sec: 82994.59833938912, Loss: 2.5099682807922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 786 , TFLOPS: 102.60418459181602, Tokens per sec: 83840.52049665795, Loss: 2.514892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 787 , TFLOPS: 102.60666494580377, Tokens per sec: 83842.54725775131, Loss: 2.5140912532806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 788 , TFLOPS: 100.84875787060152, Tokens per sec: 82406.11613404966, Loss: 2.516094923019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 789 , TFLOPS: 102.12826329795244, Tokens per sec: 83451.632956138, Loss: 2.5022590160369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 790 , TFLOPS: 102.18227743558958, Tokens per sec: 83495.76929844891, Loss: 2.5030882358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 791 , TFLOPS: 101.57318669297739, Tokens per sec: 82998.06557326988, Loss: 2.5161266326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 792 , TFLOPS: 102.71447241418953, Tokens per sec: 83930.63951538045, Loss: 2.5190649032592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 793 , TFLOPS: 100.44053564930101, Tokens per sec: 82072.54724845017, Loss: 2.53352427482605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 794 , TFLOPS: 102.79223835817518, Tokens per sec: 83994.18407008461, Loss: 2.5275282859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 795 , TFLOPS: 100.6635545007855, Tokens per sec: 82254.78169301413, Loss: 2.526475429534912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 796 , TFLOPS: 101.58583104512589, Tokens per sec: 83008.397598905, Loss: 2.5134997367858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 797 , TFLOPS: 101.5448655209443, Tokens per sec: 82974.92361454981, Loss: 2.513526201248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 798 , TFLOPS: 101.69875543675194, Tokens per sec: 83100.6710262351, Loss: 2.5021822452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 799 , TFLOPS: 102.69029165853806, Tokens per sec: 83910.8807974696, Loss: 2.5167412757873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 800 , TFLOPS: 102.76253035756758, Tokens per sec: 83969.90889803617, Loss: 2.5025291442871094 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/model.pt +[rank0]:[2024-08-27 01:49:06,956] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007818707003025338, 'preprocessing_with_comm': 0.0016082830115919933, 'state_converting': 3.298864334996324, : 3.309957731005852}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800, took 15.56s + eval ppl=9.997880935668945, eval loss=2.302373170852661 +------------------------------------------------------------------ +iteration: 801 , TFLOPS: 100.4915420130131, Tokens per sec: 82114.22586125993, Loss: 2.5028128623962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 802 , TFLOPS: 101.4647236797816, Tokens per sec: 82909.43765310125, Loss: 2.519895315170288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 803 , TFLOPS: 102.00655864531052, Tokens per sec: 83352.18495150779, Loss: 2.522951364517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 804 , TFLOPS: 101.56965234786307, Tokens per sec: 82995.17756889484, Loss: 2.4952874183654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 805 , TFLOPS: 101.0163432664096, Tokens per sec: 82543.05447499633, Loss: 2.548877239227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 806 , TFLOPS: 101.74206566501728, Tokens per sec: 83136.06092865516, Loss: 2.5018575191497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 807 , TFLOPS: 101.7547609931744, Tokens per sec: 83146.43460809918, Loss: 2.5235095024108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 808 , TFLOPS: 100.87711686248359, Tokens per sec: 82429.2889963419, Loss: 2.517184257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 809 , TFLOPS: 102.52252945635504, Tokens per sec: 83773.79798348258, Loss: 2.490805149078369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 810 , TFLOPS: 102.14463505225764, Tokens per sec: 83465.01073802721, Loss: 2.5314903259277344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 811 , TFLOPS: 102.07774644455169, Tokens per sec: 83410.35433480474, Loss: 2.514120578765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 812 , TFLOPS: 102.72471964825611, Tokens per sec: 83939.01279412353, Loss: 2.506927967071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 813 , TFLOPS: 102.82706674980045, Tokens per sec: 84022.64324544418, Loss: 2.508352518081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 814 , TFLOPS: 102.6074288044643, Tokens per sec: 83843.1714263263, Loss: 2.5019192695617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 815 , TFLOPS: 103.45113038108835, Tokens per sec: 84532.58170339656, Loss: 2.5174951553344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 816 , TFLOPS: 101.43303207319356, Tokens per sec: 82883.54162554363, Loss: 2.4983463287353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 817 , TFLOPS: 101.99573096012487, Tokens per sec: 83343.3373712132, Loss: 2.503474712371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 818 , TFLOPS: 103.33046973058693, Tokens per sec: 84433.98677979037, Loss: 2.5004289150238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 819 , TFLOPS: 102.05986031267683, Tokens per sec: 83395.7391160198, Loss: 2.515340566635132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 820 , TFLOPS: 103.17003861953283, Tokens per sec: 84302.89438908389, Loss: 2.5232105255126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 821 , TFLOPS: 102.76818809034448, Tokens per sec: 83974.53197713118, Loss: 2.522876739501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 822 , TFLOPS: 101.52396959491497, Tokens per sec: 82957.84901548226, Loss: 2.501544713973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 823 , TFLOPS: 101.96820881778763, Tokens per sec: 83320.84832022646, Loss: 2.527092456817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 824 , TFLOPS: 102.7066611598194, Tokens per sec: 83924.25673835503, Loss: 2.5131096839904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 825 , TFLOPS: 101.94977966905512, Tokens per sec: 83305.78938838863, Loss: 2.4905638694763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 826 , TFLOPS: 102.70642426643563, Tokens per sec: 83924.06316667299, Loss: 2.520542860031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 827 , TFLOPS: 100.98367439880114, Tokens per sec: 82516.35990230198, Loss: 2.5293266773223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 828 , TFLOPS: 102.3445198322449, Tokens per sec: 83628.3417372468, Loss: 2.5106797218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 829 , TFLOPS: 102.67145856538774, Tokens per sec: 83895.49179224952, Loss: 2.509485960006714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 830 , TFLOPS: 100.35880362106367, Tokens per sec: 82005.76190420786, Loss: 2.506370782852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 831 , TFLOPS: 102.74099386300577, Tokens per sec: 83952.31087393103, Loss: 2.503174304962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 832 , TFLOPS: 102.08505912588693, Tokens per sec: 83416.32971496893, Loss: 2.499911308288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 833 , TFLOPS: 101.00446359647934, Tokens per sec: 82533.34729088639, Loss: 2.520995855331421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 834 , TFLOPS: 101.57507263313985, Tokens per sec: 82999.60662352506, Loss: 2.5001869201660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 835 , TFLOPS: 102.15728816710524, Tokens per sec: 83475.34992388933, Loss: 2.512641191482544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 836 , TFLOPS: 101.05887943149929, Tokens per sec: 82577.8118704694, Loss: 2.5029213428497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 837 , TFLOPS: 101.45766180889427, Tokens per sec: 82903.6672166105, Loss: 2.5102787017822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 838 , TFLOPS: 101.69401249258117, Tokens per sec: 83096.79544445891, Loss: 2.493619918823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 839 , TFLOPS: 101.69016742711509, Tokens per sec: 83093.65354248579, Loss: 2.5050699710845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 840 , TFLOPS: 100.73107453832563, Tokens per sec: 82309.95405380856, Loss: 2.5090701580047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 841 , TFLOPS: 102.77177715993217, Tokens per sec: 83977.46469828214, Loss: 2.5118579864501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 842 , TFLOPS: 100.43232826433616, Tokens per sec: 82065.8407829185, Loss: 2.496177911758423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 843 , TFLOPS: 101.47129765856985, Tokens per sec: 82914.80942039841, Loss: 2.5225391387939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 844 , TFLOPS: 101.65659336444999, Tokens per sec: 83066.21931160889, Loss: 2.5189359188079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 845 , TFLOPS: 101.57733680605469, Tokens per sec: 83001.45673749877, Loss: 2.489450693130493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 846 , TFLOPS: 100.60125856064381, Tokens per sec: 82203.8780766842, Loss: 2.4935805797576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 847 , TFLOPS: 102.63871268021357, Tokens per sec: 83868.73428651964, Loss: 2.5222420692443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 848 , TFLOPS: 102.05778139758337, Tokens per sec: 83394.04038098089, Loss: 2.496340036392212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 849 , TFLOPS: 102.01483380075187, Tokens per sec: 83358.94679404038, Loss: 2.4940438270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 850 , TFLOPS: 102.12332609831653, Tokens per sec: 83447.59864322066, Loss: 2.483661413192749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 851 , TFLOPS: 102.90500027080675, Tokens per sec: 84086.32473164573, Loss: 2.5269925594329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 852 , TFLOPS: 102.25776330640005, Tokens per sec: 83557.45074666732, Loss: 2.5069406032562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 853 , TFLOPS: 103.33399096622048, Tokens per sec: 84436.86407207121, Loss: 2.5215952396392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 854 , TFLOPS: 101.48677394787114, Tokens per sec: 82927.45549478162, Loss: 2.4963152408599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 855 , TFLOPS: 101.9965887634745, Tokens per sec: 83344.03830441201, Loss: 2.4881672859191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 856 , TFLOPS: 103.23799925602155, Tokens per sec: 84358.42677462123, Loss: 2.4994375705718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 857 , TFLOPS: 102.78714229671806, Tokens per sec: 83990.01994708371, Loss: 2.5176029205322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 858 , TFLOPS: 102.74783701699693, Tokens per sec: 83957.90259121565, Loss: 2.5106067657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 859 , TFLOPS: 102.76610410127856, Tokens per sec: 83972.82909601876, Loss: 2.503312826156616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 860 , TFLOPS: 100.63507732061078, Tokens per sec: 82231.51225602553, Loss: 2.4928011894226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 861 , TFLOPS: 102.6783582262836, Tokens per sec: 83901.1296827806, Loss: 2.4793014526367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 862 , TFLOPS: 102.11612812024364, Tokens per sec: 83441.71699004486, Loss: 2.496803045272827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 863 , TFLOPS: 102.75241164869362, Tokens per sec: 83961.64064053653, Loss: 2.4929964542388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 864 , TFLOPS: 102.13336840026723, Tokens per sec: 83455.80446665642, Loss: 2.492324113845825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 865 , TFLOPS: 100.21976988319646, Tokens per sec: 81892.15385794974, Loss: 2.5033297538757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 866 , TFLOPS: 103.15926176527657, Tokens per sec: 84294.08834405018, Loss: 2.4797780513763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 867 , TFLOPS: 102.048651138932, Tokens per sec: 83386.5798116031, Loss: 2.4930152893066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 868 , TFLOPS: 101.01945506700649, Tokens per sec: 82545.59720736892, Loss: 2.4875051975250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 869 , TFLOPS: 101.44689005538403, Tokens per sec: 82894.86533952766, Loss: 2.4923901557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 870 , TFLOPS: 102.08245219123333, Tokens per sec: 83414.199521555, Loss: 2.4826011657714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 871 , TFLOPS: 100.90755583425192, Tokens per sec: 82454.16146374264, Loss: 2.4834866523742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 872 , TFLOPS: 101.64025726312593, Tokens per sec: 83052.87066268834, Loss: 2.492992639541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 873 , TFLOPS: 101.38551949668376, Tokens per sec: 82844.71787619495, Loss: 2.489509105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 874 , TFLOPS: 101.47392556476177, Tokens per sec: 82916.95675019614, Loss: 2.4881725311279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 875 , TFLOPS: 101.33483073404113, Tokens per sec: 82803.29878339483, Loss: 2.509410858154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 876 , TFLOPS: 101.21716931372184, Tokens per sec: 82707.15460797751, Loss: 2.50730037689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 877 , TFLOPS: 102.1391106496502, Tokens per sec: 83460.49660644591, Loss: 2.49385666847229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 878 , TFLOPS: 100.45497692612362, Tokens per sec: 82084.34758749345, Loss: 2.5111165046691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 879 , TFLOPS: 102.0528547073147, Tokens per sec: 83390.01465553808, Loss: 2.513368844985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 880 , TFLOPS: 102.10459297752931, Tokens per sec: 83432.29132799199, Loss: 2.4905991554260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 881 , TFLOPS: 100.9801995608316, Tokens per sec: 82513.52052274678, Loss: 2.5089516639709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 882 , TFLOPS: 102.6772012255527, Tokens per sec: 83900.18426769953, Loss: 2.5156755447387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 883 , TFLOPS: 101.04166997499969, Tokens per sec: 82563.74958055276, Loss: 2.519446611404419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 884 , TFLOPS: 101.56335651289642, Tokens per sec: 82990.03308007438, Loss: 2.503352165222168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 885 , TFLOPS: 102.78968751699178, Tokens per sec: 83992.09971208916, Loss: 2.501779079437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 886 , TFLOPS: 102.01571817626407, Tokens per sec: 83359.66944003725, Loss: 2.498481035232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 887 , TFLOPS: 102.65483340355057, Tokens per sec: 83881.90694454264, Loss: 2.5027036666870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 888 , TFLOPS: 102.57322717970929, Tokens per sec: 83815.22439831083, Loss: 2.5055034160614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 889 , TFLOPS: 103.33367503791456, Tokens per sec: 84436.60591891973, Loss: 2.501332998275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 890 , TFLOPS: 103.35895100069587, Tokens per sec: 84457.25955876951, Loss: 2.4935452938079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 891 , TFLOPS: 103.31860381050983, Tokens per sec: 84424.29083103943, Loss: 2.491201400756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 892 , TFLOPS: 100.12693030539992, Tokens per sec: 81816.29224902885, Loss: 2.4879229068756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 893 , TFLOPS: 102.76497464098969, Tokens per sec: 83971.90618494184, Loss: 2.485236167907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 894 , TFLOPS: 103.40136050374598, Tokens per sec: 84491.91345542949, Loss: 2.507641077041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 895 , TFLOPS: 102.77567422127396, Tokens per sec: 83980.64908742381, Loss: 2.5252034664154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 896 , TFLOPS: 101.95568889808936, Tokens per sec: 83310.6179715496, Loss: 2.508805274963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 897 , TFLOPS: 101.9448832714942, Tokens per sec: 83301.78841589713, Loss: 2.5037829875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 898 , TFLOPS: 101.52336931699027, Tokens per sec: 82957.35851293747, Loss: 2.5128207206726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 899 , TFLOPS: 103.38177207111127, Tokens per sec: 84475.90724287255, Loss: 2.492112874984741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 900 , TFLOPS: 101.511434310779, Tokens per sec: 82947.60611212783, Loss: 2.5197343826293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 901 , TFLOPS: 102.83695883248411, Tokens per sec: 84030.72632084986, Loss: 2.4940898418426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 902 , TFLOPS: 101.34645708387296, Tokens per sec: 82812.79897313128, Loss: 2.4786391258239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 903 , TFLOPS: 100.80825402214872, Tokens per sec: 82373.01939681705, Loss: 2.492049217224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 904 , TFLOPS: 103.31725639245492, Tokens per sec: 84423.18982106113, Loss: 2.4803719520568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 905 , TFLOPS: 101.31445444788937, Tokens per sec: 82786.64879544797, Loss: 2.496283531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 906 , TFLOPS: 100.92527292967648, Tokens per sec: 82468.63855849266, Loss: 2.5045158863067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 907 , TFLOPS: 102.10519361630153, Tokens per sec: 83432.7821253946, Loss: 2.5082685947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 908 , TFLOPS: 101.44529857722993, Tokens per sec: 82893.56490175966, Loss: 2.4806087017059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 909 , TFLOPS: 101.52072000325882, Tokens per sec: 82955.1936904881, Loss: 2.500887155532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 910 , TFLOPS: 100.95784983558063, Tokens per sec: 82495.25798691109, Loss: 2.5065338611602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 911 , TFLOPS: 100.684692627128, Tokens per sec: 82272.05420019235, Loss: 2.495668649673462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 912 , TFLOPS: 102.21625485687073, Tokens per sec: 83523.53312403437, Loss: 2.5094869136810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 913 , TFLOPS: 101.05405747849208, Tokens per sec: 82573.8717285391, Loss: 2.4887349605560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 914 , TFLOPS: 101.89171784969325, Tokens per sec: 83258.34558114385, Loss: 2.4862608909606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 915 , TFLOPS: 102.26314901601097, Tokens per sec: 83561.85154863089, Loss: 2.4813241958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 916 , TFLOPS: 99.73377155519873, Tokens per sec: 81495.03211343286, Loss: 2.5137667655944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 917 , TFLOPS: 102.55972771889056, Tokens per sec: 83804.19364136882, Loss: 2.476912260055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 918 , TFLOPS: 102.73608454140474, Tokens per sec: 83948.29934087364, Loss: 2.498312473297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 919 , TFLOPS: 101.46956062030364, Tokens per sec: 82913.39004171574, Loss: 2.4937851428985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 920 , TFLOPS: 102.66188329349818, Tokens per sec: 83887.66759109915, Loss: 2.5116167068481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 921 , TFLOPS: 101.68761342746151, Tokens per sec: 83091.56660362307, Loss: 2.4798126220703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 922 , TFLOPS: 101.66308847905472, Tokens per sec: 83071.52663695153, Loss: 2.5086536407470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 923 , TFLOPS: 102.58260021396549, Tokens per sec: 83822.88334588494, Loss: 2.4881367683410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 924 , TFLOPS: 102.6675539563273, Tokens per sec: 83892.30123567267, Loss: 2.5049262046813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 925 , TFLOPS: 101.40909529021201, Tokens per sec: 82863.9822639818, Loss: 2.468663454055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 926 , TFLOPS: 102.01791723276574, Tokens per sec: 83361.46634571352, Loss: 2.4851536750793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 927 , TFLOPS: 103.37807842960628, Tokens per sec: 84472.88907331604, Loss: 2.479231834411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 928 , TFLOPS: 102.59558236748205, Tokens per sec: 83833.49139771369, Loss: 2.4978864192962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 929 , TFLOPS: 103.17905664122158, Tokens per sec: 84310.26324675028, Loss: 2.4947075843811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 930 , TFLOPS: 100.03291501242776, Tokens per sec: 81739.46993297235, Loss: 2.4813008308410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 931 , TFLOPS: 103.33340640775626, Tokens per sec: 84436.38641430224, Loss: 2.4987525939941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 932 , TFLOPS: 103.28849576315608, Tokens per sec: 84399.68877049659, Loss: 2.4785759449005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 933 , TFLOPS: 102.66913335327948, Tokens per sec: 83893.59180157958, Loss: 2.5007991790771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 934 , TFLOPS: 102.85579440364342, Tokens per sec: 84046.11735091486, Loss: 2.5068113803863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 935 , TFLOPS: 102.11273038485541, Tokens per sec: 83438.94061299365, Loss: 2.5032927989959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 936 , TFLOPS: 102.03190746722325, Tokens per sec: 83372.89812642943, Loss: 2.5202231407165527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 937 , TFLOPS: 102.59931119001264, Tokens per sec: 83836.53831459187, Loss: 2.4827516078948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 938 , TFLOPS: 101.54152428462592, Tokens per sec: 82972.19340434301, Loss: 2.4963972568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 939 , TFLOPS: 102.73012937317026, Tokens per sec: 83943.43321960958, Loss: 2.482306957244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 940 , TFLOPS: 102.047384583533, Tokens per sec: 83385.54487658254, Loss: 2.4787561893463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 941 , TFLOPS: 101.5157524105645, Tokens per sec: 82951.13454262035, Loss: 2.486172676086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 942 , TFLOPS: 103.42245434126463, Tokens per sec: 84509.14977306966, Loss: 2.4809622764587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 943 , TFLOPS: 100.73382187227934, Tokens per sec: 82312.1989711048, Loss: 2.4763641357421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 944 , TFLOPS: 102.15630714329484, Tokens per sec: 83474.54830408008, Loss: 2.4722037315368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 945 , TFLOPS: 102.25091989044938, Tokens per sec: 83551.85881532883, Loss: 2.5039315223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 946 , TFLOPS: 101.39366138416183, Tokens per sec: 82851.37082204412, Loss: 2.483645439147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 947 , TFLOPS: 102.27586756438515, Tokens per sec: 83572.244201912, Loss: 2.487445116043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 948 , TFLOPS: 100.85924601955438, Tokens per sec: 82414.68627054802, Loss: 2.4879937171936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 949 , TFLOPS: 100.77383301184311, Tokens per sec: 82344.89310322072, Loss: 2.4895312786102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 950 , TFLOPS: 102.73493218170339, Tokens per sec: 83947.35771809728, Loss: 2.4380288124084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 951 , TFLOPS: 101.525031895852, Tokens per sec: 82958.71704892397, Loss: 2.4870922565460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 952 , TFLOPS: 101.63336361224209, Tokens per sec: 83047.23768309278, Loss: 2.4942362308502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 953 , TFLOPS: 102.25546391858461, Tokens per sec: 83555.57185769182, Loss: 2.4749860763549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 954 , TFLOPS: 100.97072820675743, Tokens per sec: 82505.78123551843, Loss: 2.506204605102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 955 , TFLOPS: 101.96003394121514, Tokens per sec: 83314.16841813904, Loss: 2.4689462184906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 956 , TFLOPS: 103.33357523743841, Tokens per sec: 84436.52436938159, Loss: 2.4906387329101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 957 , TFLOPS: 102.83861447015313, Tokens per sec: 84032.07918500912, Loss: 2.4864261150360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 958 , TFLOPS: 102.11222702798017, Tokens per sec: 83438.52930713333, Loss: 2.493075132369995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 959 , TFLOPS: 102.91883628314544, Tokens per sec: 84097.63049349832, Loss: 2.4956295490264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 960 , TFLOPS: 101.99474902615029, Tokens per sec: 83342.53500768534, Loss: 2.48844051361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 961 , TFLOPS: 103.4087894800098, Tokens per sec: 84497.98386317352, Loss: 2.484403610229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 962 , TFLOPS: 101.33200567730383, Tokens per sec: 82800.99035681135, Loss: 2.488056182861328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 963 , TFLOPS: 101.99224272288842, Tokens per sec: 83340.48704277209, Loss: 2.4678378105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 964 , TFLOPS: 101.99932067996893, Tokens per sec: 83346.2706237054, Loss: 2.4735236167907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 965 , TFLOPS: 103.3718908158108, Tokens per sec: 84467.8330148004, Loss: 2.5006256103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 966 , TFLOPS: 101.56750176215873, Tokens per sec: 82993.42026995482, Loss: 2.4813642501831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 967 , TFLOPS: 103.35026213199339, Tokens per sec: 84450.15966048132, Loss: 2.4776227474212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 968 , TFLOPS: 100.34542318661254, Tokens per sec: 81994.8284067748, Loss: 2.46065092086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 969 , TFLOPS: 102.68956067782442, Tokens per sec: 83910.28349431137, Loss: 2.5028998851776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 970 , TFLOPS: 102.06671490693803, Tokens per sec: 83401.34018144358, Loss: 2.486095428466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 971 , TFLOPS: 102.7465772949692, Tokens per sec: 83956.87323991867, Loss: 2.495215892791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 972 , TFLOPS: 103.35551044871471, Tokens per sec: 84454.44819517802, Loss: 2.4858598709106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 973 , TFLOPS: 100.90154298065607, Tokens per sec: 82449.24820628486, Loss: 2.486426830291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 974 , TFLOPS: 102.09381989762538, Tokens per sec: 83423.48836707875, Loss: 2.483232021331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 975 , TFLOPS: 102.06582125136485, Tokens per sec: 83400.60995247, Loss: 2.4814727306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 976 , TFLOPS: 102.00093057376617, Tokens per sec: 83347.58610936947, Loss: 2.4742517471313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 977 , TFLOPS: 102.01672786083616, Tokens per sec: 83360.4944792927, Loss: 2.47477388381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 978 , TFLOPS: 101.98806461727158, Tokens per sec: 83337.07300511855, Loss: 2.4801154136657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 979 , TFLOPS: 100.3019556812521, Tokens per sec: 81959.30998919166, Loss: 2.4668145179748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 980 , TFLOPS: 102.38764936129589, Tokens per sec: 83663.5839856872, Loss: 2.4709830284118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 981 , TFLOPS: 100.90988080723417, Tokens per sec: 82456.06125901657, Loss: 2.479536294937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 982 , TFLOPS: 102.2004771461824, Tokens per sec: 83510.64075047651, Loss: 2.4989781379699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 983 , TFLOPS: 101.61980390197878, Tokens per sec: 83036.15769477864, Loss: 2.486973762512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 984 , TFLOPS: 101.04539730565212, Tokens per sec: 82566.79527837888, Loss: 2.465028762817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 985 , TFLOPS: 101.34915393243467, Tokens per sec: 82815.00263751412, Loss: 2.4972190856933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 986 , TFLOPS: 100.59205177892511, Tokens per sec: 82196.35497833819, Loss: 2.475207805633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 987 , TFLOPS: 101.47824173093811, Tokens per sec: 82920.48360068683, Loss: 2.475447177886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 988 , TFLOPS: 102.7108634034505, Tokens per sec: 83927.69049980139, Loss: 2.4875590801239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 989 , TFLOPS: 101.01184225120633, Tokens per sec: 82539.3765796073, Loss: 2.4694459438323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 990 , TFLOPS: 102.84299296387549, Tokens per sec: 84035.65696494211, Loss: 2.472111940383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 991 , TFLOPS: 102.34180990877849, Tokens per sec: 83626.12738902276, Loss: 2.473806619644165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 992 , TFLOPS: 101.53004626875435, Tokens per sec: 82962.81442210393, Loss: 2.4939143657684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 993 , TFLOPS: 102.02314193413967, Tokens per sec: 83365.73558370194, Loss: 2.491941213607788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 994 , TFLOPS: 101.88980901205218, Tokens per sec: 83256.7858207695, Loss: 2.479351282119751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 995 , TFLOPS: 102.67246553464018, Tokens per sec: 83896.3146127474, Loss: 2.4780595302581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 996 , TFLOPS: 101.9323068201543, Tokens per sec: 83291.51187376062, Loss: 2.4755358695983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 997 , TFLOPS: 102.85928923271769, Tokens per sec: 84048.97306571636, Loss: 2.4739861488342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 998 , TFLOPS: 102.73417289823053, Tokens per sec: 83946.73728802592, Loss: 2.46413254737854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 999 , TFLOPS: 102.54299238360828, Tokens per sec: 83790.51876810381, Loss: 2.473569869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1000 , TFLOPS: 100.63995773865206, Tokens per sec: 82235.50017124019, Loss: 2.489980936050415 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/model.pt +[rank0]:[2024-08-27 02:41:55,796] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007762045992421918, 'preprocessing_with_comm': 0.0015996160072972998, 'state_converting': 3.2967784839856904, : 3.3077582719997736}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000, took 15.52s + eval ppl=9.271358489990234, eval loss=2.2269299030303955 +------------------------------------------------------------------ +iteration: 1001 , TFLOPS: 99.17346715845935, Tokens per sec: 81037.19296733878, Loss: 2.471320390701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1002 , TFLOPS: 101.03010131009302, Tokens per sec: 82554.29652665359, Loss: 2.4725894927978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1003 , TFLOPS: 101.79811414486215, Tokens per sec: 83181.85958435264, Loss: 2.4755985736846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1004 , TFLOPS: 101.79539227931579, Tokens per sec: 83179.63547794765, Loss: 2.4802942276000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1005 , TFLOPS: 100.9114870190981, Tokens per sec: 82457.37373607817, Loss: 2.4630701541900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1006 , TFLOPS: 102.8109696182806, Tokens per sec: 84009.48986490228, Loss: 2.4698374271392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1007 , TFLOPS: 102.84364305093102, Tokens per sec: 84036.18816780992, Loss: 2.4632458686828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1008 , TFLOPS: 102.8228574947833, Tokens per sec: 84019.20375481382, Loss: 2.469029664993286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1009 , TFLOPS: 103.41075185362848, Tokens per sec: 84499.58736917334, Loss: 2.4724364280700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1010 , TFLOPS: 102.55762391575361, Tokens per sec: 83802.47456966892, Loss: 2.4794135093688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1011 , TFLOPS: 103.45335290294032, Tokens per sec: 84534.3977832146, Loss: 2.469273328781128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1012 , TFLOPS: 103.41196257668337, Tokens per sec: 84500.57668214827, Loss: 2.4664220809936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1013 , TFLOPS: 100.73147584646729, Tokens per sec: 82310.28197302169, Loss: 2.4825711250305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1014 , TFLOPS: 102.48623596859836, Tokens per sec: 83744.14164035911, Loss: 2.4946086406707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1015 , TFLOPS: 103.0749215940721, Tokens per sec: 84225.17181904985, Loss: 2.479606866836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1016 , TFLOPS: 103.38182329910863, Tokens per sec: 84475.94910258793, Loss: 2.465057373046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1017 , TFLOPS: 101.23632754657346, Tokens per sec: 82722.80929321736, Loss: 2.4685416221618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1018 , TFLOPS: 103.29688370542769, Tokens per sec: 84406.54277405156, Loss: 2.470712184906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1019 , TFLOPS: 101.29629047188813, Tokens per sec: 82771.806543075, Loss: 2.486079454421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1020 , TFLOPS: 102.6902313079312, Tokens per sec: 83910.83148343518, Loss: 2.4824509620666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1021 , TFLOPS: 102.01589856883469, Tokens per sec: 83359.81684345043, Loss: 2.4760282039642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1022 , TFLOPS: 102.7314978033235, Tokens per sec: 83944.55139911432, Loss: 2.4888863563537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1023 , TFLOPS: 100.87568833651356, Tokens per sec: 82428.12171099825, Loss: 2.480642318725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1024 , TFLOPS: 102.56396034235917, Tokens per sec: 83807.65222696244, Loss: 2.457282543182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1025 , TFLOPS: 102.8581878556819, Tokens per sec: 84048.07310218844, Loss: 2.473118305206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1026 , TFLOPS: 102.87381851209082, Tokens per sec: 84060.84531390907, Loss: 2.4643325805664062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1027 , TFLOPS: 100.42533941174155, Tokens per sec: 82060.13001155439, Loss: 2.4811172485351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1028 , TFLOPS: 102.09668006135152, Tokens per sec: 83425.82548048654, Loss: 2.483236312866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1029 , TFLOPS: 103.34734134622597, Tokens per sec: 84447.77301124307, Loss: 2.471409559249878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1030 , TFLOPS: 99.91847639595802, Tokens per sec: 81645.95919354286, Loss: 2.505504608154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1031 , TFLOPS: 102.62976712133543, Tokens per sec: 83861.42464008114, Loss: 2.4814393520355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1032 , TFLOPS: 100.8969325035359, Tokens per sec: 82445.48086674571, Loss: 2.4874942302703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1033 , TFLOPS: 101.97747141341819, Tokens per sec: 83328.41702555661, Loss: 2.4719996452331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1034 , TFLOPS: 101.07718704859974, Tokens per sec: 82592.77149568217, Loss: 2.4546403884887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1035 , TFLOPS: 101.6099600993305, Tokens per sec: 83028.1140702327, Loss: 2.4776132106781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1036 , TFLOPS: 100.84055313251305, Tokens per sec: 82399.41183134883, Loss: 2.469486713409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1037 , TFLOPS: 101.23978985353779, Tokens per sec: 82725.63843336562, Loss: 2.458179235458374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1038 , TFLOPS: 100.30937593188884, Tokens per sec: 81965.3732670012, Loss: 2.4683098793029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1039 , TFLOPS: 102.63133844141429, Tokens per sec: 83862.70860616695, Loss: 2.4821345806121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1040 , TFLOPS: 100.36746148753666, Tokens per sec: 82012.83646977624, Loss: 2.4624581336975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1041 , TFLOPS: 101.22630349027567, Tokens per sec: 82714.61837877428, Loss: 2.4919209480285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1042 , TFLOPS: 101.40102837179157, Tokens per sec: 82857.39057727951, Loss: 2.4826772212982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1043 , TFLOPS: 101.01775655978702, Tokens per sec: 82544.20931339642, Loss: 2.4403789043426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1044 , TFLOPS: 102.80917843925472, Tokens per sec: 84008.02624641119, Loss: 2.4392523765563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1045 , TFLOPS: 102.19761223068248, Tokens per sec: 83508.29975427205, Loss: 2.4838664531707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1046 , TFLOPS: 102.76025497348611, Tokens per sec: 83968.04962312961, Loss: 2.471421480178833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1047 , TFLOPS: 103.33337914858781, Tokens per sec: 84436.36414013398, Loss: 2.472264528274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1048 , TFLOPS: 103.28861550284095, Tokens per sec: 84399.78661287547, Loss: 2.475785732269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1049 , TFLOPS: 103.43006003793492, Tokens per sec: 84515.364583607, Loss: 2.4683189392089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1050 , TFLOPS: 103.4414708340515, Tokens per sec: 84524.68863885388, Loss: 2.453117847442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1051 , TFLOPS: 100.46460518179677, Tokens per sec: 82092.21508305734, Loss: 2.479893922805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1052 , TFLOPS: 102.46441471398525, Tokens per sec: 83726.31093148567, Loss: 2.47659969329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1053 , TFLOPS: 103.43289449079124, Tokens per sec: 84517.68068800122, Loss: 2.471841812133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1054 , TFLOPS: 103.43356961433717, Tokens per sec: 84518.23234882983, Loss: 2.4703962802886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1055 , TFLOPS: 101.55638508836086, Tokens per sec: 82984.33654962594, Loss: 2.4759137630462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1056 , TFLOPS: 103.34972499360485, Tokens per sec: 84449.72075087689, Loss: 2.471745014190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1057 , TFLOPS: 102.0783484548176, Tokens per sec: 83410.84625289014, Loss: 2.457975387573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1058 , TFLOPS: 101.93686597270866, Tokens per sec: 83295.23727467583, Loss: 2.476597547531128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1059 , TFLOPS: 102.16773726105478, Tokens per sec: 83483.88814753912, Loss: 2.4690184593200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1060 , TFLOPS: 102.696732172891, Tokens per sec: 83916.14350754098, Loss: 2.434955596923828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1061 , TFLOPS: 101.2449095614759, Tokens per sec: 82729.82187851486, Loss: 2.469024181365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1062 , TFLOPS: 102.19662924839001, Tokens per sec: 83507.49653413666, Loss: 2.4624805450439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1063 , TFLOPS: 102.6815044560168, Tokens per sec: 83903.70054808677, Loss: 2.48750638961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1064 , TFLOPS: 102.12265093989149, Tokens per sec: 83447.04695389143, Loss: 2.486764669418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1065 , TFLOPS: 100.86031891733762, Tokens per sec: 82415.56296294555, Loss: 2.4819729328155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1066 , TFLOPS: 102.28227022665914, Tokens per sec: 83577.47598207527, Loss: 2.4666497707366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1067 , TFLOPS: 102.79673497394293, Tokens per sec: 83997.85837058161, Loss: 2.468642473220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1068 , TFLOPS: 99.7337709853685, Tokens per sec: 81495.0316478099, Loss: 2.4961016178131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1069 , TFLOPS: 103.29580536023708, Tokens per sec: 84405.66163043726, Loss: 2.465024948120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1070 , TFLOPS: 100.23152650568193, Tokens per sec: 81901.76049682501, Loss: 2.4652040004730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1071 , TFLOPS: 102.02914702833726, Tokens per sec: 83370.64250075567, Loss: 2.462350368499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1072 , TFLOPS: 101.50618603622914, Tokens per sec: 82943.31761189055, Loss: 2.4771504402160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1073 , TFLOPS: 100.4357438112586, Tokens per sec: 82068.63171423285, Loss: 2.4576144218444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1074 , TFLOPS: 102.23903548220218, Tokens per sec: 83542.14775941816, Loss: 2.4596152305603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1075 , TFLOPS: 99.68891646185035, Tokens per sec: 81458.37986198507, Loss: 2.4677114486694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1076 , TFLOPS: 101.51569603189505, Tokens per sec: 82951.08847415823, Loss: 2.470202922821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1077 , TFLOPS: 101.77588194362309, Tokens per sec: 83163.69308041243, Loss: 2.4473047256469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1078 , TFLOPS: 101.55051905428013, Tokens per sec: 82979.54326217344, Loss: 2.454349994659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1079 , TFLOPS: 101.69164523100208, Tokens per sec: 83094.86109408391, Loss: 2.483562469482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1080 , TFLOPS: 101.54863806546253, Tokens per sec: 82978.0062577896, Loss: 2.4482860565185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1081 , TFLOPS: 101.04319080347312, Tokens per sec: 82564.992288648, Loss: 2.4697864055633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1082 , TFLOPS: 102.83672710074889, Tokens per sec: 84030.5369668837, Loss: 2.454092502593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1083 , TFLOPS: 100.62251161704437, Tokens per sec: 82221.24449617142, Loss: 2.4376258850097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1084 , TFLOPS: 102.70107455019019, Tokens per sec: 83919.69177581488, Loss: 2.454826593399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1085 , TFLOPS: 103.29290477572336, Tokens per sec: 84403.29148816339, Loss: 2.457460403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1086 , TFLOPS: 103.37589795017266, Tokens per sec: 84471.10734744028, Loss: 2.4583070278167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1087 , TFLOPS: 103.46870647959626, Tokens per sec: 84546.9435859364, Loss: 2.4572954177856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1088 , TFLOPS: 103.36029118264642, Tokens per sec: 84458.35465594065, Loss: 2.465998411178589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1089 , TFLOPS: 100.17421742717636, Tokens per sec: 81854.93177351118, Loss: 2.4857945442199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1090 , TFLOPS: 102.52810426546026, Tokens per sec: 83778.35330351081, Loss: 2.4672205448150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1091 , TFLOPS: 103.36958483317338, Tokens per sec: 84465.94873702608, Loss: 2.449193000793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1092 , TFLOPS: 103.28276167630197, Tokens per sec: 84395.00330051956, Loss: 2.4879088401794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1093 , TFLOPS: 100.13873343572011, Tokens per sec: 81825.93689065309, Loss: 2.473304271697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1094 , TFLOPS: 103.39068964103885, Tokens per sec: 84483.19401881905, Loss: 2.4423341751098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1095 , TFLOPS: 102.06578724745911, Tokens per sec: 83400.58216700327, Loss: 2.4915807247161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1096 , TFLOPS: 102.57390868913194, Tokens per sec: 83815.7812772037, Loss: 2.4646527767181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1097 , TFLOPS: 101.5874548627538, Tokens per sec: 83009.72446208957, Loss: 2.4656476974487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1098 , TFLOPS: 102.75361568525945, Tokens per sec: 83962.62448980911, Loss: 2.473311424255371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1099 , TFLOPS: 101.63825919829814, Tokens per sec: 83051.23799248278, Loss: 2.456650733947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1100 , TFLOPS: 100.8144501483123, Tokens per sec: 82378.08241100781, Loss: 2.4755287170410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1101 , TFLOPS: 103.39029401446004, Tokens per sec: 84482.87074215717, Loss: 2.44803524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1102 , TFLOPS: 101.25365226608459, Tokens per sec: 82736.96576750788, Loss: 2.4660377502441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1103 , TFLOPS: 100.9881607975407, Tokens per sec: 82520.02585420225, Loss: 2.4813601970672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1104 , TFLOPS: 100.8869797515978, Tokens per sec: 82437.34821693048, Loss: 2.446561336517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1105 , TFLOPS: 103.43747068913018, Tokens per sec: 84521.42001746579, Loss: 2.4502980709075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1106 , TFLOPS: 99.35342696353084, Tokens per sec: 81184.2427566402, Loss: 2.4645276069641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1107 , TFLOPS: 103.18264550353558, Tokens per sec: 84313.19579853259, Loss: 2.438912868499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1108 , TFLOPS: 99.74761259726773, Tokens per sec: 81506.34198520756, Loss: 2.4430360794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1109 , TFLOPS: 101.58605039290327, Tokens per sec: 83008.57683362042, Loss: 2.4721012115478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1110 , TFLOPS: 100.95208078863755, Tokens per sec: 82490.54395014542, Loss: 2.453437566757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1111 , TFLOPS: 99.64220963263581, Tokens per sec: 81420.21450949312, Loss: 2.487313747406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1112 , TFLOPS: 102.17157189178074, Tokens per sec: 83487.0215230175, Loss: 2.440185070037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1113 , TFLOPS: 100.5000062271628, Tokens per sec: 82121.1421885299, Loss: 2.4735991954803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1114 , TFLOPS: 100.86174498057213, Tokens per sec: 82416.7282359246, Loss: 2.455310344696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1115 , TFLOPS: 102.56968448989909, Tokens per sec: 83812.32957526998, Loss: 2.4810614585876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1116 , TFLOPS: 101.57413258114906, Tokens per sec: 82998.83848284428, Loss: 2.4634411334991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1117 , TFLOPS: 101.68953806200973, Tokens per sec: 83093.13927205691, Loss: 2.481936454772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1118 , TFLOPS: 101.61132549705829, Tokens per sec: 83029.22977186456, Loss: 2.4822208881378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1119 , TFLOPS: 100.98715117827896, Tokens per sec: 82519.20086831358, Loss: 2.4607696533203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1120 , TFLOPS: 102.8592330915429, Tokens per sec: 84048.92719131724, Loss: 2.470834732055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1121 , TFLOPS: 100.2098761954276, Tokens per sec: 81884.06947098763, Loss: 2.449192523956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1122 , TFLOPS: 102.82209315128794, Tokens per sec: 84018.5791900678, Loss: 2.4310390949249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1123 , TFLOPS: 102.80806773436333, Tokens per sec: 84007.11866085234, Loss: 2.468939781188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1124 , TFLOPS: 102.65908891176231, Tokens per sec: 83885.38422984898, Loss: 2.462512493133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1125 , TFLOPS: 102.29457092963185, Tokens per sec: 83587.52720312252, Loss: 2.4790477752685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1126 , TFLOPS: 102.88425988140352, Tokens per sec: 84069.37722555923, Loss: 2.45436429977417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1127 , TFLOPS: 101.40401773560521, Tokens per sec: 82859.83326340471, Loss: 2.4365763664245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1128 , TFLOPS: 102.63792541842658, Tokens per sec: 83868.09099464757, Loss: 2.455136775970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1129 , TFLOPS: 101.94246199299172, Tokens per sec: 83299.8099269035, Loss: 2.448432207107544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1130 , TFLOPS: 103.42508779626009, Tokens per sec: 84511.30163693761, Loss: 2.454648733139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1131 , TFLOPS: 102.1901148024127, Tokens per sec: 83502.17341263183, Loss: 2.4464592933654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1132 , TFLOPS: 101.48754338000418, Tokens per sec: 82928.08421758447, Loss: 2.4537808895111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1133 , TFLOPS: 102.04022365409479, Tokens per sec: 83379.69349679985, Loss: 2.442667245864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1134 , TFLOPS: 101.91299606747863, Tokens per sec: 83275.73256064634, Loss: 2.4343161582946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1135 , TFLOPS: 101.28027576457028, Tokens per sec: 82758.72051347034, Loss: 2.4709408283233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1136 , TFLOPS: 101.64413781880903, Tokens per sec: 83056.04156463139, Loss: 2.4587137699127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1137 , TFLOPS: 102.15423402017194, Tokens per sec: 83472.8543018094, Loss: 2.4727118015289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1138 , TFLOPS: 101.15217924233656, Tokens per sec: 82654.04954765533, Loss: 2.454646348953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1139 , TFLOPS: 103.41514604578794, Tokens per sec: 84503.1779767034, Loss: 2.4361038208007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1140 , TFLOPS: 101.50451451335543, Tokens per sec: 82941.95176752162, Loss: 2.454209089279175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1141 , TFLOPS: 100.11223868530237, Tokens per sec: 81804.2873480512, Loss: 2.445671796798706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1142 , TFLOPS: 101.67633363827767, Tokens per sec: 83082.34959751334, Loss: 2.451019763946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1143 , TFLOPS: 101.81894210949014, Tokens per sec: 83198.87865040968, Loss: 2.4814226627349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1144 , TFLOPS: 101.05948917690905, Tokens per sec: 82578.31010913996, Loss: 2.444023609161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1145 , TFLOPS: 101.47298882058466, Tokens per sec: 82916.19131241509, Loss: 2.4669554233551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1146 , TFLOPS: 100.87964435609857, Tokens per sec: 82431.35427643817, Loss: 2.4530692100524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1147 , TFLOPS: 102.00621248197103, Tokens per sec: 83351.90209253207, Loss: 2.447965383529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1148 , TFLOPS: 101.65779961612682, Tokens per sec: 83067.2049709057, Loss: 2.4437289237976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1149 , TFLOPS: 102.32562135814587, Tokens per sec: 83612.89930757048, Loss: 2.466353416442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1150 , TFLOPS: 100.96529566491327, Tokens per sec: 82501.3421657315, Loss: 2.4486072063446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1151 , TFLOPS: 101.61205891091686, Tokens per sec: 83029.82906320813, Loss: 2.4477357864379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1152 , TFLOPS: 101.56949414171335, Tokens per sec: 82995.04829457731, Loss: 2.4261879920959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1153 , TFLOPS: 101.32136916666629, Tokens per sec: 82792.29899016126, Loss: 2.457207202911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1154 , TFLOPS: 101.57851522193607, Tokens per sec: 83002.41965145056, Loss: 2.4611361026763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1155 , TFLOPS: 102.82629183390591, Tokens per sec: 84022.01004171865, Loss: 2.4674315452575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1156 , TFLOPS: 101.3830182449678, Tokens per sec: 82842.6740390299, Loss: 2.4763777256011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1157 , TFLOPS: 101.0635503314924, Tokens per sec: 82581.62858309344, Loss: 2.465872287750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1158 , TFLOPS: 102.68627483469623, Tokens per sec: 83907.59854730622, Loss: 2.450124740600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1159 , TFLOPS: 102.16756099877132, Tokens per sec: 83483.74411908991, Loss: 2.4542720317840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1160 , TFLOPS: 101.3877207573857, Tokens per sec: 82846.51658298, Loss: 2.48343563079834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1161 , TFLOPS: 102.7388828206231, Tokens per sec: 83950.58588685708, Loss: 2.4183895587921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1162 , TFLOPS: 102.48549405466544, Tokens per sec: 83743.53540338596, Loss: 2.4524989128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1163 , TFLOPS: 102.61949666883494, Tokens per sec: 83853.03238895803, Loss: 2.4363186359405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1164 , TFLOPS: 102.03554337281655, Tokens per sec: 83375.86911848624, Loss: 2.453155279159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1165 , TFLOPS: 102.03623812329765, Tokens per sec: 83376.43681698882, Loss: 2.45627760887146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1166 , TFLOPS: 101.52199823257762, Tokens per sec: 82956.23816456903, Loss: 2.450596332550049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1167 , TFLOPS: 102.08968680314389, Tokens per sec: 83420.1111091826, Loss: 2.442934274673462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1168 , TFLOPS: 102.81881137720659, Tokens per sec: 84015.89756798554, Loss: 2.453030586242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1169 , TFLOPS: 102.60519390046002, Tokens per sec: 83841.34522873287, Loss: 2.4523143768310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1170 , TFLOPS: 100.28880311039316, Tokens per sec: 81948.56268496554, Loss: 2.454871892929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1171 , TFLOPS: 103.32108338862533, Tokens per sec: 84426.31695814761, Loss: 2.456571578979492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1172 , TFLOPS: 102.15687963616423, Tokens per sec: 83475.01610274092, Loss: 2.4575605392456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1173 , TFLOPS: 101.54386355596498, Tokens per sec: 82974.10488317233, Loss: 2.4509520530700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1174 , TFLOPS: 101.67896983796213, Tokens per sec: 83084.50370413733, Loss: 2.495234251022339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1175 , TFLOPS: 102.98432980134622, Tokens per sec: 84151.14693317332, Loss: 2.4650678634643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1176 , TFLOPS: 100.26594628094064, Tokens per sec: 81929.88578123273, Loss: 2.4585015773773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1177 , TFLOPS: 102.70909967211152, Tokens per sec: 83926.24930952182, Loss: 2.447035789489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1178 , TFLOPS: 102.13509267636132, Tokens per sec: 83457.21341703973, Loss: 2.4266159534454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1179 , TFLOPS: 101.51386618867724, Tokens per sec: 82949.5932621606, Loss: 2.430222988128662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1180 , TFLOPS: 102.397663270486, Tokens per sec: 83671.76660866741, Loss: 2.4506306648254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1181 , TFLOPS: 101.13810486381269, Tokens per sec: 82642.54901065674, Loss: 2.4574899673461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1182 , TFLOPS: 102.79329137530229, Tokens per sec: 83995.04451748625, Loss: 2.443164825439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1183 , TFLOPS: 100.28372442670556, Tokens per sec: 81944.41276178745, Loss: 2.4549102783203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1184 , TFLOPS: 102.79026607805737, Tokens per sec: 83992.57246922934, Loss: 2.4762160778045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1185 , TFLOPS: 100.78027904020271, Tokens per sec: 82350.16031892887, Loss: 2.453916072845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1186 , TFLOPS: 101.58783498698195, Tokens per sec: 83010.03507138163, Loss: 2.440225124359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1187 , TFLOPS: 102.6533995474723, Tokens per sec: 83880.73530383034, Loss: 2.472256660461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1188 , TFLOPS: 101.13695683222865, Tokens per sec: 82641.61092449646, Loss: 2.444472312927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1189 , TFLOPS: 101.50909293318118, Tokens per sec: 82945.69291222024, Loss: 2.447340726852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1190 , TFLOPS: 102.79396406512257, Tokens per sec: 83995.59418965496, Loss: 2.457099199295044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1191 , TFLOPS: 101.28666529760267, Tokens per sec: 82763.94156539232, Loss: 2.4819395542144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1192 , TFLOPS: 102.71209958168275, Tokens per sec: 83928.700612857, Loss: 2.437889575958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1193 , TFLOPS: 103.29157416157453, Tokens per sec: 84402.20420908934, Loss: 2.462435722351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1194 , TFLOPS: 101.96324868377219, Tokens per sec: 83316.79526703723, Loss: 2.4316771030426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1195 , TFLOPS: 101.40104331830828, Tokens per sec: 82857.40279046306, Loss: 2.455841064453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1196 , TFLOPS: 103.26525105235135, Tokens per sec: 84380.69491892595, Loss: 2.4437382221221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1197 , TFLOPS: 102.2123183011307, Tokens per sec: 83520.31646300321, Loss: 2.4604039192199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1198 , TFLOPS: 100.82101472651775, Tokens per sec: 82383.44649684675, Loss: 2.4601151943206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1199 , TFLOPS: 102.14540879955041, Tokens per sec: 83465.64298685818, Loss: 2.4369335174560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1200 , TFLOPS: 102.79995011368176, Tokens per sec: 84000.4855440273, Loss: 2.4699084758758545 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/model.pt +[rank0]:[2024-08-27 03:34:47,126] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0077010390086798, 'preprocessing_with_comm': 0.0015049319918034598, 'state_converting': 3.3068605930020567, : 3.3176867910078727}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200, took 15.49s + eval ppl=8.689545631408691, eval loss=2.1621205806732178 +------------------------------------------------------------------ +iteration: 1201 , TFLOPS: 100.48763221515357, Tokens per sec: 82111.03106478175, Loss: 2.454911231994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1202 , TFLOPS: 102.79582667942732, Tokens per sec: 83997.11617975135, Loss: 2.446805477142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1203 , TFLOPS: 102.1227886274976, Tokens per sec: 83447.15946197878, Loss: 2.454878807067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1204 , TFLOPS: 102.69056879019092, Tokens per sec: 83911.10724887716, Loss: 2.453547716140747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1205 , TFLOPS: 100.69828984271095, Tokens per sec: 82283.16483506864, Loss: 2.458956718444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1206 , TFLOPS: 102.1672080664042, Tokens per sec: 83483.45572896746, Loss: 2.4579551219940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1207 , TFLOPS: 102.74032985390795, Tokens per sec: 83951.76829500408, Loss: 2.447817802429199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1208 , TFLOPS: 102.84619016099138, Tokens per sec: 84038.2694770087, Loss: 2.467710018157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1209 , TFLOPS: 101.31472014386668, Tokens per sec: 82786.86590247063, Loss: 2.4655508995056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1210 , TFLOPS: 102.64603508206649, Tokens per sec: 83874.71760956905, Loss: 2.4545223712921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1211 , TFLOPS: 103.21824263939891, Tokens per sec: 84342.2831345983, Loss: 2.4591064453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1212 , TFLOPS: 103.35996053524558, Tokens per sec: 84458.0844754378, Loss: 2.440263509750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1213 , TFLOPS: 102.5617359268871, Tokens per sec: 83805.83459981867, Loss: 2.4302845001220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1214 , TFLOPS: 101.77575277023203, Tokens per sec: 83163.58752950939, Loss: 2.44590163230896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1215 , TFLOPS: 101.3898313849368, Tokens per sec: 82848.24123108438, Loss: 2.4428417682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1216 , TFLOPS: 102.37684916820538, Tokens per sec: 83654.75886989126, Loss: 2.45340633392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1217 , TFLOPS: 102.1507235405456, Tokens per sec: 83469.98579853894, Loss: 2.450330972671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1218 , TFLOPS: 103.37393906206799, Tokens per sec: 84469.5066895437, Loss: 2.448810338973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1219 , TFLOPS: 101.66828267969377, Tokens per sec: 83075.77095200433, Loss: 2.457826614379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1220 , TFLOPS: 100.7146092601085, Tokens per sec: 82296.49985112337, Loss: 2.440988302230835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1221 , TFLOPS: 103.43267766921514, Tokens per sec: 84517.5035175099, Loss: 2.467970848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1222 , TFLOPS: 102.74344997520451, Tokens per sec: 83954.31782643456, Loss: 2.4722695350646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1223 , TFLOPS: 100.8701202472723, Tokens per sec: 82423.57188194385, Loss: 2.442689895629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1224 , TFLOPS: 101.9727041339007, Tokens per sec: 83324.52155874218, Loss: 2.4312515258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1225 , TFLOPS: 102.18365425839649, Tokens per sec: 83496.894335807, Loss: 2.4383790493011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1226 , TFLOPS: 101.30678218786237, Tokens per sec: 82780.37959427835, Loss: 2.4502158164978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1227 , TFLOPS: 102.29898321040208, Tokens per sec: 83591.13259132211, Loss: 2.462719440460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1228 , TFLOPS: 100.9550554261843, Tokens per sec: 82492.97460305886, Loss: 2.454742908477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1229 , TFLOPS: 101.55353098434884, Tokens per sec: 82982.00438776654, Loss: 2.4346137046813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1230 , TFLOPS: 101.9117143578109, Tokens per sec: 83274.68524268271, Loss: 2.428833484649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1231 , TFLOPS: 100.28041041414693, Tokens per sec: 81941.70479681552, Loss: 2.4479310512542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1232 , TFLOPS: 102.7950127542363, Tokens per sec: 83996.45110052545, Loss: 2.4660401344299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1233 , TFLOPS: 99.32762250427287, Tokens per sec: 81163.15726871464, Loss: 2.4466352462768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1234 , TFLOPS: 102.83503275733497, Tokens per sec: 84029.15247526392, Loss: 2.464412212371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1235 , TFLOPS: 102.68920189020677, Tokens per sec: 83909.99031971298, Loss: 2.473782777786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1236 , TFLOPS: 100.24484874207306, Tokens per sec: 81912.64643912508, Loss: 2.437790632247925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1237 , TFLOPS: 103.49156080565488, Tokens per sec: 84565.61844408159, Loss: 2.4515044689178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1238 , TFLOPS: 100.56926783582836, Tokens per sec: 82177.73763192295, Loss: 2.4374806880950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1239 , TFLOPS: 100.82700413898392, Tokens per sec: 82388.34059995404, Loss: 2.4678075313568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1240 , TFLOPS: 101.27257857504074, Tokens per sec: 82752.43094176248, Loss: 2.4444754123687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1241 , TFLOPS: 102.13711537782673, Tokens per sec: 83458.86621847609, Loss: 2.449875831604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1242 , TFLOPS: 102.05482177289724, Tokens per sec: 83391.62199546235, Loss: 2.451172113418579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1243 , TFLOPS: 101.77605599259118, Tokens per sec: 83163.83530030468, Loss: 2.4675350189208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1244 , TFLOPS: 102.28041558884838, Tokens per sec: 83575.96050977748, Loss: 2.4507248401641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1245 , TFLOPS: 102.80381266206642, Tokens per sec: 84003.64173174324, Loss: 2.4572594165802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1246 , TFLOPS: 102.10854112282645, Tokens per sec: 83435.5174591486, Loss: 2.458279609680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1247 , TFLOPS: 101.90576169750021, Tokens per sec: 83269.82117070773, Loss: 2.434330463409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1248 , TFLOPS: 102.52532932516702, Tokens per sec: 83776.08582836388, Loss: 2.4525837898254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1249 , TFLOPS: 103.46216162374796, Tokens per sec: 84541.59561574318, Loss: 2.4649219512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1250 , TFLOPS: 103.45185762007716, Tokens per sec: 84533.17594909511, Loss: 2.4545459747314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1251 , TFLOPS: 99.82047881174091, Tokens per sec: 81565.88284479681, Loss: 2.4426939487457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1252 , TFLOPS: 102.2627639645764, Tokens per sec: 83561.5369131916, Loss: 2.439868450164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1253 , TFLOPS: 101.28625648032944, Tokens per sec: 82763.60751027447, Loss: 2.4451160430908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1254 , TFLOPS: 103.40830256119958, Tokens per sec: 84497.58598927927, Loss: 2.4520761966705322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1255 , TFLOPS: 101.45946129530412, Tokens per sec: 82905.13762327889, Loss: 2.4428775310516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1256 , TFLOPS: 103.45492073366123, Tokens per sec: 84535.67889805621, Loss: 2.4205708503723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1257 , TFLOPS: 102.78730472026005, Tokens per sec: 83990.15266754103, Loss: 2.4461734294891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1258 , TFLOPS: 100.0316437235035, Tokens per sec: 81738.43113006646, Loss: 2.439723253250122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1259 , TFLOPS: 103.29629823113721, Tokens per sec: 84406.06436793736, Loss: 2.4449119567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1260 , TFLOPS: 103.22950685219645, Tokens per sec: 84351.48741284182, Loss: 2.435222625732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1261 , TFLOPS: 100.70026718170362, Tokens per sec: 82284.7805696608, Loss: 2.458085775375366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1262 , TFLOPS: 102.63718485619755, Tokens per sec: 83867.48586218646, Loss: 2.454153537750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1263 , TFLOPS: 101.54487558428957, Tokens per sec: 82974.93183756829, Loss: 2.4582154750823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1264 , TFLOPS: 102.07169424263674, Tokens per sec: 83405.4089248225, Loss: 2.438725709915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1265 , TFLOPS: 101.74358335769757, Tokens per sec: 83137.30107441313, Loss: 2.4299116134643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1266 , TFLOPS: 101.41550794209597, Tokens per sec: 82869.22220691253, Loss: 2.442606210708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1267 , TFLOPS: 101.35249921218768, Tokens per sec: 82817.73615171546, Loss: 2.4416773319244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1268 , TFLOPS: 102.08894928241169, Tokens per sec: 83419.50846200685, Loss: 2.447758197784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1269 , TFLOPS: 100.18309284061377, Tokens per sec: 81862.18410230437, Loss: 2.4413843154907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1270 , TFLOPS: 102.82942980974642, Tokens per sec: 84024.5741625566, Loss: 2.426546812057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1271 , TFLOPS: 99.13554607939842, Tokens per sec: 81006.20667746269, Loss: 2.43263578414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1272 , TFLOPS: 102.14907313330914, Tokens per sec: 83468.63720830077, Loss: 2.4342761039733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1273 , TFLOPS: 102.78532320701818, Tokens per sec: 83988.5335220624, Loss: 2.443119525909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1274 , TFLOPS: 100.36662572298606, Tokens per sec: 82012.15354504745, Loss: 2.4304544925689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1275 , TFLOPS: 103.37774452016603, Tokens per sec: 84472.6162273168, Loss: 2.447739362716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1276 , TFLOPS: 100.72202050970031, Tokens per sec: 82302.55577394737, Loss: 2.4535789489746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1277 , TFLOPS: 100.87257210367085, Tokens per sec: 82425.5753569235, Loss: 2.452324867248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1278 , TFLOPS: 102.82640615172528, Tokens per sec: 84022.10345375165, Loss: 2.4347727298736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1279 , TFLOPS: 102.12432079839257, Tokens per sec: 83448.41143825866, Loss: 2.4331672191619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1280 , TFLOPS: 102.10443415243742, Tokens per sec: 83432.16154792088, Loss: 2.4261980056762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1281 , TFLOPS: 100.91526635056294, Tokens per sec: 82460.46192510672, Loss: 2.4220335483551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1282 , TFLOPS: 102.12454675833747, Tokens per sec: 83448.59607594636, Loss: 2.4418323040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1283 , TFLOPS: 102.84878732111852, Tokens per sec: 84040.39168340543, Loss: 2.462263345718384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1284 , TFLOPS: 102.75857765646175, Tokens per sec: 83966.67904421102, Loss: 2.4470572471618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1285 , TFLOPS: 102.09080300680577, Tokens per sec: 83421.02318792837, Loss: 2.4338138103485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1286 , TFLOPS: 102.60153437061702, Tokens per sec: 83838.35493269346, Loss: 2.4217164516448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1287 , TFLOPS: 103.39058276661831, Tokens per sec: 84483.10668897883, Loss: 2.4388439655303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1288 , TFLOPS: 102.65534531391752, Tokens per sec: 83882.32523968122, Loss: 2.413576126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1289 , TFLOPS: 103.32729759668725, Tokens per sec: 84431.39474752295, Loss: 2.4447383880615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1290 , TFLOPS: 101.96624665015477, Tokens per sec: 83319.24498254298, Loss: 2.437922954559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1291 , TFLOPS: 101.32118751353575, Tokens per sec: 82792.15055671213, Loss: 2.4296624660491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1292 , TFLOPS: 102.576343162391, Tokens per sec: 83817.7705479726, Loss: 2.4406826496124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1293 , TFLOPS: 102.0746693039642, Tokens per sec: 83407.83992401806, Loss: 2.4462790489196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1294 , TFLOPS: 103.29984988913982, Tokens per sec: 84408.9665191191, Loss: 2.4491424560546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1295 , TFLOPS: 102.76822877328817, Tokens per sec: 83974.56522021173, Loss: 2.41755747795105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1296 , TFLOPS: 100.0626307950552, Tokens per sec: 81763.75146390953, Loss: 2.4317166805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1297 , TFLOPS: 103.43795781604977, Tokens per sec: 84521.81806141156, Loss: 2.448307991027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1298 , TFLOPS: 103.2220972418143, Tokens per sec: 84345.43282945821, Loss: 2.429741144180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1299 , TFLOPS: 100.29323040882983, Tokens per sec: 81952.18034448693, Loss: 2.473726987838745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1300 , TFLOPS: 102.70394660256167, Tokens per sec: 83922.03860374083, Loss: 2.4569597244262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1301 , TFLOPS: 102.11849777363585, Tokens per sec: 83443.65329483194, Loss: 2.432835578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1302 , TFLOPS: 101.20775984677975, Tokens per sec: 82699.46589031791, Loss: 2.4316067695617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1303 , TFLOPS: 102.1991858275949, Tokens per sec: 83509.58558081719, Loss: 2.4409079551696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1304 , TFLOPS: 101.55721031790787, Tokens per sec: 82985.0108659321, Loss: 2.4359660148620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1305 , TFLOPS: 101.38430935908796, Tokens per sec: 82843.72904161371, Loss: 2.456599473953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1306 , TFLOPS: 101.95114412904407, Tokens per sec: 83306.90432376994, Loss: 2.4202239513397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1307 , TFLOPS: 100.25433081680649, Tokens per sec: 81920.39448648007, Loss: 2.445788860321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1308 , TFLOPS: 102.74297486388146, Tokens per sec: 83953.92960074205, Loss: 2.4409561157226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1309 , TFLOPS: 98.83656384859883, Tokens per sec: 80761.90060018902, Loss: 2.444335460662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1310 , TFLOPS: 102.5797851576913, Tokens per sec: 83820.58309093732, Loss: 2.4405336380004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1311 , TFLOPS: 101.90128277758781, Tokens per sec: 83266.16132994964, Loss: 2.4265880584716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1312 , TFLOPS: 99.75265901918608, Tokens per sec: 81510.4655464635, Loss: 2.4574594497680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1313 , TFLOPS: 103.46062770663045, Tokens per sec: 84540.34221257991, Loss: 2.4436116218566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1314 , TFLOPS: 99.92821398906374, Tokens per sec: 81653.91603153749, Loss: 2.4305553436279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1315 , TFLOPS: 100.88387943497902, Tokens per sec: 82434.81486841202, Loss: 2.442131996154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1316 , TFLOPS: 102.78758136902525, Tokens per sec: 83990.37872436864, Loss: 2.418884515762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1317 , TFLOPS: 101.66752625734489, Tokens per sec: 83075.152859831, Loss: 2.4427459239959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1318 , TFLOPS: 102.04450829558905, Tokens per sec: 83383.1945876613, Loss: 2.427495002746582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1319 , TFLOPS: 101.44982250450421, Tokens per sec: 82897.26151919173, Loss: 2.431471347808838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1320 , TFLOPS: 102.1540940835483, Tokens per sec: 83472.73995599196, Loss: 2.437598943710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1321 , TFLOPS: 102.86966645814852, Tokens per sec: 84057.45256374942, Loss: 2.4500834941864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1322 , TFLOPS: 102.79320697539508, Tokens per sec: 83994.97555214942, Loss: 2.4459574222564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1323 , TFLOPS: 101.27881453057378, Tokens per sec: 82757.52650155629, Loss: 2.4392833709716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1324 , TFLOPS: 102.68955455026648, Tokens per sec: 83910.27848732604, Loss: 2.4535844326019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1325 , TFLOPS: 103.4298343203753, Tokens per sec: 84515.18014397856, Loss: 2.425934314727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1326 , TFLOPS: 103.39797400508053, Tokens per sec: 84489.14626019372, Loss: 2.434887409210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1327 , TFLOPS: 103.24466565229395, Tokens per sec: 84363.87405862371, Loss: 2.4325618743896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1328 , TFLOPS: 102.02133432513459, Tokens per sec: 83364.25853985189, Loss: 2.4440977573394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1329 , TFLOPS: 101.14767459528944, Tokens per sec: 82650.3686845932, Loss: 2.4411942958831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1330 , TFLOPS: 103.36510650205378, Tokens per sec: 84462.28937738576, Loss: 2.4390640258789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1331 , TFLOPS: 101.41894078019035, Tokens per sec: 82872.02726728856, Loss: 2.4489731788635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1332 , TFLOPS: 103.3408244319299, Tokens per sec: 84442.44787281145, Loss: 2.453040838241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1333 , TFLOPS: 102.56793986102117, Tokens per sec: 83810.9039941031, Loss: 2.4221835136413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1334 , TFLOPS: 99.9010366644351, Tokens per sec: 81631.70873997685, Loss: 2.440753221511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1335 , TFLOPS: 103.29604319926167, Tokens per sec: 84405.85597482676, Loss: 2.4234883785247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1336 , TFLOPS: 103.41884159043076, Tokens per sec: 84506.19770136282, Loss: 2.4247891902923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1337 , TFLOPS: 100.9594675478542, Tokens per sec: 82496.57986125346, Loss: 2.448028087615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1338 , TFLOPS: 101.90197767340253, Tokens per sec: 83266.72914720807, Loss: 2.416138172149658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1339 , TFLOPS: 102.17157654042535, Tokens per sec: 83487.02532154464, Loss: 2.4351859092712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1340 , TFLOPS: 102.07315573392394, Tokens per sec: 83406.60314697541, Loss: 2.42688250541687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1341 , TFLOPS: 101.40056379527297, Tokens per sec: 82857.0109598469, Loss: 2.4343695640563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1342 , TFLOPS: 101.67020268767408, Tokens per sec: 83077.33983995044, Loss: 2.430570602416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1343 , TFLOPS: 102.02698982253472, Tokens per sec: 83368.87979236298, Loss: 2.4246089458465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1344 , TFLOPS: 101.42992839340356, Tokens per sec: 82881.00552889139, Loss: 2.418657064437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1345 , TFLOPS: 100.89748312502256, Tokens per sec: 82445.93079373674, Loss: 2.440577507019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1346 , TFLOPS: 102.06041051127748, Tokens per sec: 83396.18869745977, Loss: 2.4197418689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1347 , TFLOPS: 99.3953824576347, Tokens per sec: 81218.52567090298, Loss: 2.4376049041748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1348 , TFLOPS: 102.13014017373364, Tokens per sec: 83453.16659965388, Loss: 2.4312171936035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1349 , TFLOPS: 102.57189456579536, Tokens per sec: 83814.13548517723, Loss: 2.430622100830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1350 , TFLOPS: 99.63330606773337, Tokens per sec: 81412.9391774132, Loss: 2.4517464637756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1351 , TFLOPS: 103.41074545322923, Tokens per sec: 84499.58213924234, Loss: 2.4388325214385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1352 , TFLOPS: 99.77021961724098, Tokens per sec: 81524.81476318416, Loss: 2.423652410507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1353 , TFLOPS: 101.07223275436485, Tokens per sec: 82588.72321432797, Loss: 2.4361045360565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1354 , TFLOPS: 102.16126311996943, Tokens per sec: 83478.59796019907, Loss: 2.4380300045013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1355 , TFLOPS: 101.4594898060189, Tokens per sec: 82905.16092011785, Loss: 2.42622447013855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1356 , TFLOPS: 102.11278227880311, Tokens per sec: 83438.98301687419, Loss: 2.422062635421753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1357 , TFLOPS: 101.09800753269849, Tokens per sec: 82609.7844492062, Loss: 2.4245357513427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1358 , TFLOPS: 102.04355046824942, Tokens per sec: 83382.41192228542, Loss: 2.4288294315338135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1359 , TFLOPS: 102.95380270207889, Tokens per sec: 84126.20245452454, Loss: 2.427424192428589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1360 , TFLOPS: 102.86515189415768, Tokens per sec: 84053.76359729731, Loss: 2.440598249435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1361 , TFLOPS: 101.97899633175054, Tokens per sec: 83329.66307558087, Loss: 2.4417362213134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1362 , TFLOPS: 102.62326323299402, Tokens per sec: 83856.11014549239, Loss: 2.433589220046997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1363 , TFLOPS: 103.31896248870969, Tokens per sec: 84424.58391622972, Loss: 2.4466633796691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1364 , TFLOPS: 102.66677831142616, Tokens per sec: 83891.66743625706, Loss: 2.4379470348358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1365 , TFLOPS: 103.4295694009218, Tokens per sec: 84514.96367147348, Loss: 2.433318614959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1366 , TFLOPS: 101.79959136397753, Tokens per sec: 83183.06665811881, Loss: 2.411118507385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1367 , TFLOPS: 101.44967220964476, Tokens per sec: 82897.13870939313, Loss: 2.4132349491119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1368 , TFLOPS: 103.39286192953665, Tokens per sec: 84484.96905167028, Loss: 2.4470744132995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1369 , TFLOPS: 101.30431548648511, Tokens per sec: 82778.36398908444, Loss: 2.4176342487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1370 , TFLOPS: 103.24497912450029, Tokens per sec: 84364.130204833, Loss: 2.4366652965545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1371 , TFLOPS: 102.74945070942198, Tokens per sec: 83959.2211808363, Loss: 2.4300169944763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1372 , TFLOPS: 100.02420443009797, Tokens per sec: 81732.35229192034, Loss: 2.4329023361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1373 , TFLOPS: 103.2846984408017, Tokens per sec: 84396.58588064906, Loss: 2.461564779281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1374 , TFLOPS: 103.31360017257809, Tokens per sec: 84420.2022296803, Loss: 2.4253859519958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1375 , TFLOPS: 100.94444168332085, Tokens per sec: 82484.30184054308, Loss: 2.440256357192993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1376 , TFLOPS: 101.99781664596759, Tokens per sec: 83345.04163880562, Loss: 2.4565939903259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1377 , TFLOPS: 102.06859052048341, Tokens per sec: 83402.8727935541, Loss: 2.430511236190796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1378 , TFLOPS: 100.86075903612033, Tokens per sec: 82415.92259533257, Loss: 2.4220006465911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1379 , TFLOPS: 102.2582742861104, Tokens per sec: 83557.86828134244, Loss: 2.459974765777588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1380 , TFLOPS: 101.59253100638567, Tokens per sec: 83013.87230972272, Loss: 2.4293386936187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1381 , TFLOPS: 101.67981280448709, Tokens per sec: 83085.19251378474, Loss: 2.4285800457000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1382 , TFLOPS: 102.0589850775638, Tokens per sec: 83395.02393887838, Loss: 2.448423385620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1383 , TFLOPS: 100.29307908582517, Tokens per sec: 81952.05669456444, Loss: 2.4122812747955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1384 , TFLOPS: 102.84384162014238, Tokens per sec: 84036.35042382416, Loss: 2.4210004806518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1385 , TFLOPS: 99.0978626622019, Tokens per sec: 80975.41458721395, Loss: 2.461254119873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1386 , TFLOPS: 102.6962100101877, Tokens per sec: 83915.71683495445, Loss: 2.447500705718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1387 , TFLOPS: 102.03008641834734, Tokens per sec: 83371.41010051494, Loss: 2.4163317680358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1388 , TFLOPS: 99.55178521816039, Tokens per sec: 81346.32639269374, Loss: 2.4116337299346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1389 , TFLOPS: 103.20642665289589, Tokens per sec: 84332.62798785613, Loss: 2.430541753768921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1390 , TFLOPS: 99.7170868814178, Tokens per sec: 81481.39863698476, Loss: 2.4325783252716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1391 , TFLOPS: 100.91583470977652, Tokens per sec: 82460.92634605101, Loss: 2.4350061416625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1392 , TFLOPS: 102.72223266990797, Tokens per sec: 83936.98062009474, Loss: 2.422607898712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1393 , TFLOPS: 101.89797129254254, Tokens per sec: 83263.45543027394, Loss: 2.4005353450775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1394 , TFLOPS: 101.97729716064657, Tokens per sec: 83328.27463913127, Loss: 2.4330837726593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1395 , TFLOPS: 101.60585426354359, Tokens per sec: 83024.75908611761, Loss: 2.435899496078491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1396 , TFLOPS: 102.28413911008717, Tokens per sec: 83579.00309483365, Loss: 2.4365367889404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1397 , TFLOPS: 102.75105278766168, Tokens per sec: 83960.53028020712, Loss: 2.4298970699310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1398 , TFLOPS: 102.84073542933072, Tokens per sec: 84033.81227535149, Loss: 2.440493583679199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1399 , TFLOPS: 101.19333210402684, Tokens per sec: 82687.67661031136, Loss: 2.4161906242370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1400 , TFLOPS: 102.67949397819633, Tokens per sec: 83902.05773490536, Loss: 2.4511420726776123 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/model.pt +[rank0]:[2024-08-27 04:27:37,444] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00767999301024247, 'preprocessing_with_comm': 0.0016077820037025958, 'state_converting': 3.3126816820004024, : 3.3236430910037598}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400, took 15.51s + eval ppl=9.353032112121582, eval loss=2.2357006072998047 +------------------------------------------------------------------ +iteration: 1401 , TFLOPS: 101.76252260744128, Tokens per sec: 83152.77682291837, Loss: 2.4506676197052 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1402 , TFLOPS: 100.2760386498687, Tokens per sec: 81938.13251568454, Loss: 2.442142963409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1403 , TFLOPS: 101.50193693285146, Tokens per sec: 82939.845560139, Loss: 2.411763906478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1404 , TFLOPS: 102.17361365834817, Tokens per sec: 83488.68990303947, Loss: 2.4110984802246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1405 , TFLOPS: 99.65118962415895, Tokens per sec: 81427.55229173222, Loss: 2.4443042278289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1406 , TFLOPS: 102.02160973793399, Tokens per sec: 83364.48358674004, Loss: 2.457777500152588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1407 , TFLOPS: 101.5610070413142, Tokens per sec: 82988.11326636377, Loss: 2.446122646331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1408 , TFLOPS: 101.56571891022938, Tokens per sec: 82991.96345574866, Loss: 2.4302749633789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1409 , TFLOPS: 102.56478617952872, Tokens per sec: 83808.32703977263, Loss: 2.426685333251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1410 , TFLOPS: 103.40631460767098, Tokens per sec: 84495.9615812766, Loss: 2.4391579627990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1411 , TFLOPS: 102.66794141881715, Tokens per sec: 83892.6178412473, Loss: 2.44852614402771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1412 , TFLOPS: 103.37471283670199, Tokens per sec: 84470.13896071586, Loss: 2.4247560501098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1413 , TFLOPS: 102.11925184817025, Tokens per sec: 83444.26946854561, Loss: 2.4171361923217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1414 , TFLOPS: 102.47354537970061, Tokens per sec: 83733.77183348594, Loss: 2.4372172355651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1415 , TFLOPS: 102.60332298326695, Tokens per sec: 83839.81645413271, Loss: 2.4205844402313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1416 , TFLOPS: 101.85800917412465, Tokens per sec: 83230.80135459825, Loss: 2.4204206466674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1417 , TFLOPS: 102.67782345298129, Tokens per sec: 83900.692705749, Loss: 2.406846523284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1418 , TFLOPS: 103.2688466507166, Tokens per sec: 84383.63297490912, Loss: 2.423879861831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1419 , TFLOPS: 102.77342458820085, Tokens per sec: 83978.81085432907, Loss: 2.4288604259490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1420 , TFLOPS: 103.3232755257922, Tokens per sec: 84428.1082098573, Loss: 2.436527967453003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1421 , TFLOPS: 102.65726544497142, Tokens per sec: 83883.89422819392, Loss: 2.4208641052246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1422 , TFLOPS: 101.99384196863782, Tokens per sec: 83341.79382764216, Loss: 2.4348623752593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1423 , TFLOPS: 101.35088651161774, Tokens per sec: 82816.41837256517, Loss: 2.4311275482177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1424 , TFLOPS: 103.3633269575103, Tokens per sec: 84460.83526572972, Loss: 2.4257616996765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1425 , TFLOPS: 102.6181019700178, Tokens per sec: 83851.89274465179, Loss: 2.4217138290405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1426 , TFLOPS: 102.09851400553129, Tokens per sec: 83427.32404348574, Loss: 2.432403326034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1427 , TFLOPS: 102.23696056896691, Tokens per sec: 83540.4522944006, Loss: 2.4122793674468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1428 , TFLOPS: 103.36418635250322, Tokens per sec: 84461.53749950125, Loss: 2.430211305618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1429 , TFLOPS: 101.87867696561128, Tokens per sec: 83247.68953905825, Loss: 2.433466911315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1430 , TFLOPS: 100.94204597780318, Tokens per sec: 82482.34424789365, Loss: 2.422574520111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1431 , TFLOPS: 102.65662452473704, Tokens per sec: 83883.37051577166, Loss: 2.4283740520477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1432 , TFLOPS: 102.7973865627166, Tokens per sec: 83998.39080054198, Loss: 2.417202949523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1433 , TFLOPS: 101.10783087657954, Tokens per sec: 82617.81135636677, Loss: 2.4396145343780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1434 , TFLOPS: 101.31603549414018, Tokens per sec: 82787.94070903925, Loss: 2.4039223194122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1435 , TFLOPS: 102.20300457153243, Tokens per sec: 83512.70597479175, Loss: 2.4522247314453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1436 , TFLOPS: 101.63843688813583, Tokens per sec: 83051.3831874233, Loss: 2.426926374435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1437 , TFLOPS: 102.05848745300474, Tokens per sec: 83394.61731704118, Loss: 2.427865505218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1438 , TFLOPS: 100.8022604896788, Tokens per sec: 82368.12192714863, Loss: 2.4284772872924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1439 , TFLOPS: 102.16838432632639, Tokens per sec: 83484.4168812295, Loss: 2.4192113876342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1440 , TFLOPS: 101.50622159997545, Tokens per sec: 82943.34667194316, Loss: 2.3995792865753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1441 , TFLOPS: 100.84197191078599, Tokens per sec: 82400.57115159824, Loss: 2.420356273651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1442 , TFLOPS: 102.14080426898767, Tokens per sec: 83461.8805064042, Loss: 2.41342830657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1443 , TFLOPS: 100.2730335426181, Tokens per sec: 81935.67696519174, Loss: 2.419280529022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1444 , TFLOPS: 101.30103023642958, Tokens per sec: 82775.6795266942, Loss: 2.43237566947937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1445 , TFLOPS: 101.99327796185185, Tokens per sec: 83341.33296317852, Loss: 2.4472720623016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1446 , TFLOPS: 100.77300009213685, Tokens per sec: 82344.21250308747, Loss: 2.4150960445404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1447 , TFLOPS: 103.33467443572289, Tokens per sec: 84437.4225525957, Loss: 2.437455892562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1448 , TFLOPS: 103.29147790447361, Tokens per sec: 84402.12555493427, Loss: 2.402360200881958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1449 , TFLOPS: 102.726271531511, Tokens per sec: 83940.28087788011, Loss: 2.422664165496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1450 , TFLOPS: 103.42060320926579, Tokens per sec: 84507.63716546104, Loss: 2.426365375518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1451 , TFLOPS: 101.911962722762, Tokens per sec: 83274.88818807773, Loss: 2.424616575241089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1452 , TFLOPS: 101.98453111310509, Tokens per sec: 83334.18568790363, Loss: 2.412839651107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1453 , TFLOPS: 102.0824591616274, Tokens per sec: 83414.20521724343, Loss: 2.4047627449035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1454 , TFLOPS: 101.43850601835761, Tokens per sec: 82888.01452704908, Loss: 2.424241542816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1455 , TFLOPS: 102.55565954105346, Tokens per sec: 83800.86942853392, Loss: 2.393324613571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1456 , TFLOPS: 102.51047649323164, Tokens per sec: 83763.94919704358, Loss: 2.417625904083252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1457 , TFLOPS: 102.13412987335461, Tokens per sec: 83456.4266859182, Loss: 2.4313440322875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1458 , TFLOPS: 102.76161026338922, Tokens per sec: 83969.1570653977, Loss: 2.4229183197021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1459 , TFLOPS: 102.69450430671542, Tokens per sec: 83914.32306073853, Loss: 2.4208993911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1460 , TFLOPS: 101.38778406185723, Tokens per sec: 82846.56831069328, Loss: 2.4562463760375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1461 , TFLOPS: 101.57723110291774, Tokens per sec: 83001.37036474452, Loss: 2.43344783782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1462 , TFLOPS: 103.27123180631577, Tokens per sec: 84385.58194694859, Loss: 2.4361205101013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1463 , TFLOPS: 102.22099807346919, Tokens per sec: 83527.40892841823, Loss: 2.4161365032196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1464 , TFLOPS: 101.5286947581421, Tokens per sec: 82961.71006799168, Loss: 2.4368574619293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1465 , TFLOPS: 101.32587028164257, Tokens per sec: 82795.9769670774, Loss: 2.4096901416778564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1466 , TFLOPS: 101.95797563282301, Tokens per sec: 83312.48652136624, Loss: 2.4445600509643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1467 , TFLOPS: 102.7084291437378, Tokens per sec: 83925.70140352672, Loss: 2.4239675998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1468 , TFLOPS: 100.47034727054046, Tokens per sec: 82096.90709158433, Loss: 2.4383835792541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1469 , TFLOPS: 102.10265341795831, Tokens per sec: 83430.70646393782, Loss: 2.413123607635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1470 , TFLOPS: 102.2763528609542, Tokens per sec: 83572.64075023124, Loss: 2.4161911010742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1471 , TFLOPS: 100.0838857598346, Tokens per sec: 81781.11944278225, Loss: 2.413966178894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1472 , TFLOPS: 100.88442987600311, Tokens per sec: 82435.26464794241, Loss: 2.4265329837799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1473 , TFLOPS: 101.50730287745003, Tokens per sec: 82944.23021160217, Loss: 2.4161787033081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1474 , TFLOPS: 100.99804935874614, Tokens per sec: 82528.10605211744, Loss: 2.424710512161255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1475 , TFLOPS: 101.68496824037248, Tokens per sec: 83089.40515315943, Loss: 2.4216601848602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1476 , TFLOPS: 99.38665257378642, Tokens per sec: 81211.39225808335, Loss: 2.397068738937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1477 , TFLOPS: 101.63590661399897, Tokens per sec: 83049.31563529102, Loss: 2.419912099838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1478 , TFLOPS: 101.49149389807296, Tokens per sec: 82931.31228759386, Loss: 2.449925184249878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1479 , TFLOPS: 100.79098207547783, Tokens per sec: 82358.90604457281, Loss: 2.412280559539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1480 , TFLOPS: 102.1664277024536, Tokens per sec: 83482.81807349506, Loss: 2.413742780685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1481 , TFLOPS: 100.56922183270609, Tokens per sec: 82177.70004158752, Loss: 2.405674934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1482 , TFLOPS: 102.17703754780402, Tokens per sec: 83491.48765125261, Loss: 2.4158973693847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1483 , TFLOPS: 101.70580263802096, Tokens per sec: 83106.4294758031, Loss: 2.427504539489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1484 , TFLOPS: 101.10125040734977, Tokens per sec: 82612.43428556279, Loss: 2.441016674041748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1485 , TFLOPS: 102.63834878907451, Tokens per sec: 83868.43694170252, Loss: 2.434924602508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1486 , TFLOPS: 102.5961220600151, Tokens per sec: 83833.93239437537, Loss: 2.4134511947631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1487 , TFLOPS: 103.36063573839547, Tokens per sec: 84458.63620131285, Loss: 2.4222452640533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1488 , TFLOPS: 102.11150609798509, Tokens per sec: 83437.94021667595, Loss: 2.426725387573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1489 , TFLOPS: 101.54470506092856, Tokens per sec: 82974.79249854035, Loss: 2.4208014011383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1490 , TFLOPS: 102.83696315973621, Tokens per sec: 84030.72985675895, Loss: 2.4300594329833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1491 , TFLOPS: 102.04022360870448, Tokens per sec: 83379.69345971027, Loss: 2.4053256511688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1492 , TFLOPS: 100.86597879732976, Tokens per sec: 82420.1877965854, Loss: 2.402092695236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1493 , TFLOPS: 102.48283707525778, Tokens per sec: 83741.3643171152, Loss: 2.4360568523406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1494 , TFLOPS: 102.56656433196152, Tokens per sec: 83809.78001389917, Loss: 2.4156830310821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1495 , TFLOPS: 101.89300320119392, Tokens per sec: 83259.39587494294, Loss: 2.4163248538970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1496 , TFLOPS: 102.19607570680945, Tokens per sec: 83507.04422106176, Loss: 2.408393383026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1497 , TFLOPS: 101.39319257881805, Tokens per sec: 82850.98774912959, Loss: 2.4236366748809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1498 , TFLOPS: 102.09294772816592, Tokens per sec: 83422.77569496179, Loss: 2.399383068084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1499 , TFLOPS: 101.51086321479933, Tokens per sec: 82947.13945490155, Loss: 2.418550968170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1500 , TFLOPS: 102.71556016264731, Tokens per sec: 83931.52834264643, Loss: 2.4062373638153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1501 , TFLOPS: 102.63284803735375, Tokens per sec: 83863.94213586954, Loss: 2.4290401935577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1502 , TFLOPS: 101.01101643691857, Tokens per sec: 82538.7017854944, Loss: 2.419858932495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1503 , TFLOPS: 101.5780892265749, Tokens per sec: 83002.07155967485, Loss: 2.41772723197937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1504 , TFLOPS: 102.76256184793901, Tokens per sec: 83969.93462962931, Loss: 2.4225144386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1505 , TFLOPS: 102.27210983930074, Tokens per sec: 83569.17366801309, Loss: 2.420107364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1506 , TFLOPS: 98.02573450850385, Tokens per sec: 80099.35107379331, Loss: 2.428180456161499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1507 , TFLOPS: 101.74311919588573, Tokens per sec: 83136.92179584807, Loss: 2.409987449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1508 , TFLOPS: 102.24450425029835, Tokens per sec: 83546.61643060815, Loss: 2.394803524017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1509 , TFLOPS: 100.83450894278155, Tokens per sec: 82394.47296832793, Loss: 2.4046483039855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1510 , TFLOPS: 101.54916456448792, Tokens per sec: 82978.43647369655, Loss: 2.4182231426239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1511 , TFLOPS: 101.98213677554062, Tokens per sec: 83332.22921304396, Loss: 2.4101781845092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1512 , TFLOPS: 100.2800862761625, Tokens per sec: 81941.43993532438, Loss: 2.4267706871032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1513 , TFLOPS: 102.23915515120444, Tokens per sec: 83542.24554404046, Loss: 2.4185056686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1514 , TFLOPS: 101.08612243948996, Tokens per sec: 82600.07283359599, Loss: 2.4244461059570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1515 , TFLOPS: 101.70224808096711, Tokens per sec: 83103.52495573198, Loss: 2.4199061393737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1516 , TFLOPS: 101.55566540173916, Tokens per sec: 82983.74847516097, Loss: 2.4288341999053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1517 , TFLOPS: 100.3524051098788, Tokens per sec: 82000.53351600637, Loss: 2.4140732288360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1518 , TFLOPS: 101.54926094147505, Tokens per sec: 82978.5152258137, Loss: 2.437547206878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1519 , TFLOPS: 100.4683828231155, Tokens per sec: 82095.30189102402, Loss: 2.410242795944214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1520 , TFLOPS: 102.1799925542378, Tokens per sec: 83493.90226307829, Loss: 2.4302000999450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1521 , TFLOPS: 101.63343038251932, Tokens per sec: 83047.29224280512, Loss: 2.426722288131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1522 , TFLOPS: 101.05844664585173, Tokens per sec: 82577.4582301759, Loss: 2.4200446605682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1523 , TFLOPS: 103.30074211034851, Tokens per sec: 84409.69557603654, Loss: 2.4311962127685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1524 , TFLOPS: 101.97074872899331, Tokens per sec: 83322.92374705576, Loss: 2.4251222610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1525 , TFLOPS: 103.48254363323798, Tokens per sec: 84558.25028037708, Loss: 2.4155397415161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1526 , TFLOPS: 103.41585649996705, Tokens per sec: 84503.75850710187, Loss: 2.4079575538635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1527 , TFLOPS: 101.34858499989228, Tokens per sec: 82814.53774808811, Loss: 2.4113872051239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1528 , TFLOPS: 103.33077796534674, Tokens per sec: 84434.2386463473, Loss: 2.4111621379852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1529 , TFLOPS: 101.37669707140134, Tokens per sec: 82837.50884538724, Loss: 2.406259059906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1530 , TFLOPS: 101.54567851547256, Tokens per sec: 82975.58793330721, Loss: 2.4223408699035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1531 , TFLOPS: 102.64339588102452, Tokens per sec: 83872.56105045858, Loss: 2.432647228240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1532 , TFLOPS: 102.54001045752895, Tokens per sec: 83788.08215954278, Loss: 2.3965601921081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1533 , TFLOPS: 102.10997457125481, Tokens per sec: 83436.68876675965, Loss: 2.398268461227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1534 , TFLOPS: 101.99599964533218, Tokens per sec: 83343.55692081265, Loss: 2.414081573486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1535 , TFLOPS: 101.37312289310792, Tokens per sec: 82834.58829229618, Loss: 2.410928249359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1536 , TFLOPS: 101.93658317352198, Tokens per sec: 83295.0061921805, Loss: 2.4369237422943115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1537 , TFLOPS: 100.94479671507075, Tokens per sec: 82484.59194612528, Loss: 2.403099775314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1538 , TFLOPS: 102.68977427024255, Tokens per sec: 83910.45802617459, Loss: 2.420058012008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1539 , TFLOPS: 102.04140002684429, Tokens per sec: 83380.65474125599, Loss: 2.396573305130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1540 , TFLOPS: 100.91957410052024, Tokens per sec: 82463.98189848819, Loss: 2.4177842140197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1541 , TFLOPS: 101.64828579108855, Tokens per sec: 83059.43097955929, Loss: 2.434644937515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1542 , TFLOPS: 102.8575045599906, Tokens per sec: 84047.51476368948, Loss: 2.415652275085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1543 , TFLOPS: 102.27179369342119, Tokens per sec: 83568.91533707659, Loss: 2.4196043014526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1544 , TFLOPS: 101.01856515262041, Tokens per sec: 82544.87003541562, Loss: 2.412022352218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1545 , TFLOPS: 101.50042504831998, Tokens per sec: 82938.61016036896, Loss: 2.433518886566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1546 , TFLOPS: 102.29101199120879, Tokens per sec: 83584.61910292183, Loss: 2.430988311767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1547 , TFLOPS: 100.99435307783452, Tokens per sec: 82525.08572583381, Loss: 2.396864175796509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1548 , TFLOPS: 100.83302634239202, Tokens per sec: 82393.26149738411, Loss: 2.412867784500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1549 , TFLOPS: 102.07902016562511, Tokens per sec: 83411.39512508233, Loss: 2.4210190773010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1550 , TFLOPS: 100.9723681751819, Tokens per sec: 82507.1212959346, Loss: 2.399898052215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1551 , TFLOPS: 102.19386489136828, Tokens per sec: 83505.23770685332, Loss: 2.4021549224853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1552 , TFLOPS: 100.39935279899866, Tokens per sec: 82038.89568132724, Loss: 2.4083025455474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1553 , TFLOPS: 101.59101701974075, Tokens per sec: 83012.63519226166, Loss: 2.430140495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1554 , TFLOPS: 100.89204370271814, Tokens per sec: 82441.48610174861, Loss: 2.4094913005828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1555 , TFLOPS: 101.67844397530871, Tokens per sec: 83084.07400822631, Loss: 2.41732120513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1556 , TFLOPS: 101.51834030796272, Tokens per sec: 82953.24918020284, Loss: 2.413292646408081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1557 , TFLOPS: 100.6662018603668, Tokens per sec: 82256.9449186771, Loss: 2.422520637512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1558 , TFLOPS: 102.32117608048587, Tokens per sec: 83609.26695676286, Loss: 2.4233736991882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1559 , TFLOPS: 101.06107822881827, Tokens per sec: 82579.60856436082, Loss: 2.400818347930908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1560 , TFLOPS: 101.77414773800913, Tokens per sec: 83162.27601636284, Loss: 2.4125335216522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1561 , TFLOPS: 102.63051309353064, Tokens per sec: 83862.03419316487, Loss: 2.4098541736602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1562 , TFLOPS: 102.11142570379549, Tokens per sec: 83437.8745245141, Loss: 2.431058883666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1563 , TFLOPS: 103.25241162781712, Tokens per sec: 84370.20349462284, Loss: 2.4302451610565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1564 , TFLOPS: 102.66226621674748, Tokens per sec: 83887.98048754348, Loss: 2.4188263416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1565 , TFLOPS: 101.47323093461048, Tokens per sec: 82916.38915001816, Loss: 2.4038524627685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1566 , TFLOPS: 103.34844957153159, Tokens per sec: 84448.67857066855, Loss: 2.4167909622192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1567 , TFLOPS: 101.38952114409926, Tokens per sec: 82847.98772530972, Loss: 2.4100165367126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1568 , TFLOPS: 101.985292765312, Tokens per sec: 83334.80805352856, Loss: 2.38417649269104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1569 , TFLOPS: 101.3377536254948, Tokens per sec: 82805.68715324353, Loss: 2.401026487350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1570 , TFLOPS: 102.12075429554042, Tokens per sec: 83445.49715696821, Loss: 2.404076337814331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1571 , TFLOPS: 102.08253371451114, Tokens per sec: 83414.26613632392, Loss: 2.4330623149871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1572 , TFLOPS: 101.22649376347168, Tokens per sec: 82714.7738559008, Loss: 2.422250509262085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1573 , TFLOPS: 102.7982007559311, Tokens per sec: 83999.05609877675, Loss: 2.4165608882904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1574 , TFLOPS: 100.79993911239764, Tokens per sec: 82366.22507001463, Loss: 2.416177272796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1575 , TFLOPS: 100.57574351260713, Tokens per sec: 82183.02907411764, Loss: 2.4417850971221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1576 , TFLOPS: 102.85214685482366, Tokens per sec: 84043.13684487798, Loss: 2.425380229949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1577 , TFLOPS: 102.95004955378157, Tokens per sec: 84123.13566043616, Loss: 2.4444029331207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1578 , TFLOPS: 100.38844291932618, Tokens per sec: 82029.98093780158, Loss: 2.40427565574646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1579 , TFLOPS: 102.13083523865377, Tokens per sec: 83453.73455509268, Loss: 2.4141275882720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1580 , TFLOPS: 102.19205308141314, Tokens per sec: 83503.75723029884, Loss: 2.4194467067718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1581 , TFLOPS: 103.3862941651624, Tokens per sec: 84479.60236233078, Loss: 2.412482976913452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1582 , TFLOPS: 99.79539778919973, Tokens per sec: 81545.38849563546, Loss: 2.4268839359283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1583 , TFLOPS: 102.10680908114222, Tokens per sec: 83434.10216330168, Loss: 2.4173226356506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1584 , TFLOPS: 102.16534515664733, Tokens per sec: 83481.93349744956, Loss: 2.4132540225982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1585 , TFLOPS: 101.03402690561173, Tokens per sec: 82557.50423180574, Loss: 2.425529956817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1586 , TFLOPS: 101.55897014379072, Tokens per sec: 82986.44886496292, Loss: 2.4396724700927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1587 , TFLOPS: 101.94273108367432, Tokens per sec: 83300.02980782725, Loss: 2.4163384437561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1588 , TFLOPS: 101.48484842409292, Tokens per sec: 82925.88209973504, Loss: 2.412975788116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1589 , TFLOPS: 101.00242017334469, Tokens per sec: 82531.67755724068, Loss: 2.4224815368652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1590 , TFLOPS: 102.0376556591196, Tokens per sec: 83377.59512199955, Loss: 2.4258193969726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1591 , TFLOPS: 101.00940268322174, Tokens per sec: 82537.383145807, Loss: 2.408339500427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1592 , TFLOPS: 102.12618954610693, Tokens per sec: 83449.93844012191, Loss: 2.410062551498413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1593 , TFLOPS: 101.02235344342694, Tokens per sec: 82547.96554535291, Loss: 2.4177029132843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1594 , TFLOPS: 101.39037106020896, Tokens per sec: 82848.68221364176, Loss: 2.4305601119995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1595 , TFLOPS: 101.0666035621239, Tokens per sec: 82584.12345643959, Loss: 2.4089040756225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1596 , TFLOPS: 102.24479194439819, Tokens per sec: 83546.85151286304, Loss: 2.3981845378875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1597 , TFLOPS: 101.7919037472055, Tokens per sec: 83176.78490855753, Loss: 2.417985439300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1598 , TFLOPS: 101.11496630189369, Tokens per sec: 82623.64189607315, Loss: 2.4188432693481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1599 , TFLOPS: 102.06666116535696, Tokens per sec: 83401.29626781426, Loss: 2.39795184135437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1600 , TFLOPS: 103.43066513891509, Tokens per sec: 84515.85902719459, Loss: 2.4117724895477295 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/model.pt +[rank0]:[2024-08-27 05:20:30,809] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007754627004032955, 'preprocessing_with_comm': 0.0016229579923674464, 'state_converting': 3.3346152909944067, : 3.3455916029924992}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600, took 15.64s + eval ppl=8.92743968963623, eval loss=2.189129590988159 +------------------------------------------------------------------ +iteration: 1601 , TFLOPS: 101.00587862891433, Tokens per sec: 82534.5035503152, Loss: 2.407789468765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1602 , TFLOPS: 100.26106789207834, Tokens per sec: 81925.89952411246, Loss: 2.415121078491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1603 , TFLOPS: 102.85313408758468, Tokens per sec: 84043.94353818064, Loss: 2.4150991439819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1604 , TFLOPS: 100.28293301048414, Tokens per sec: 81943.76607522006, Loss: 2.416067123413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1605 , TFLOPS: 101.03134559368257, Tokens per sec: 82555.31326280533, Loss: 2.4127817153930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1606 , TFLOPS: 102.94843292089243, Tokens per sec: 84121.81466808662, Loss: 2.4380106925964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1607 , TFLOPS: 101.49839385233459, Tokens per sec: 82936.95041784165, Loss: 2.4098477363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1608 , TFLOPS: 101.0195479623809, Tokens per sec: 82545.67311457067, Loss: 2.401442050933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1609 , TFLOPS: 103.38993202463732, Tokens per sec: 84482.57495095457, Loss: 2.4152944087982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1610 , TFLOPS: 102.62084529039008, Tokens per sec: 83854.13438234733, Loss: 2.4082140922546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1611 , TFLOPS: 102.65313353045262, Tokens per sec: 83880.51793447568, Loss: 2.414480209350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1612 , TFLOPS: 103.28029193579151, Tokens per sec: 84392.9852119717, Loss: 2.4136502742767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1613 , TFLOPS: 102.47587335017299, Tokens per sec: 83735.67407808622, Loss: 2.411696434020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1614 , TFLOPS: 102.57928779573871, Tokens per sec: 83820.17668368261, Loss: 2.4184134006500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1615 , TFLOPS: 102.12647754895308, Tokens per sec: 83450.17377466134, Loss: 2.397185802459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1616 , TFLOPS: 102.40705455727704, Tokens per sec: 83679.44047085759, Loss: 2.3936500549316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1617 , TFLOPS: 101.3802527845851, Tokens per sec: 82840.4143101619, Loss: 2.3962793350219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1618 , TFLOPS: 102.6658680448909, Tokens per sec: 83890.92363403876, Loss: 2.411024332046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1619 , TFLOPS: 101.9577362437373, Tokens per sec: 83312.29091038191, Loss: 2.414297342300415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1620 , TFLOPS: 103.35168116784904, Tokens per sec: 84451.31919120823, Loss: 2.409395933151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1621 , TFLOPS: 100.81483054204735, Tokens per sec: 82378.3932405209, Loss: 2.424976348876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1622 , TFLOPS: 102.77357376561098, Tokens per sec: 83978.93275103104, Loss: 2.4075558185577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1623 , TFLOPS: 102.0777258790498, Tokens per sec: 83410.33753020373, Loss: 2.429227590560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1624 , TFLOPS: 100.84283789821859, Tokens per sec: 82401.27877222192, Loss: 2.424457550048828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1625 , TFLOPS: 102.80762160135097, Tokens per sec: 84006.75411408347, Loss: 2.394298791885376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1626 , TFLOPS: 101.66246056272159, Tokens per sec: 83071.01355035172, Loss: 2.412252426147461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1627 , TFLOPS: 101.50387728007347, Tokens per sec: 82941.43106780307, Loss: 2.396515130996704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1628 , TFLOPS: 101.52461805146872, Tokens per sec: 82958.3788860251, Loss: 2.4289188385009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1629 , TFLOPS: 102.87269328061187, Tokens per sec: 84059.92585830158, Loss: 2.424267292022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1630 , TFLOPS: 101.38399913266636, Tokens per sec: 82843.47554761868, Loss: 2.4171125888824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1631 , TFLOPS: 101.51780012323968, Tokens per sec: 82952.80778136007, Loss: 2.431020975112915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1632 , TFLOPS: 102.08549822445885, Tokens per sec: 83416.68851371545, Loss: 2.4053328037261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1633 , TFLOPS: 102.3826269051049, Tokens per sec: 83659.48000744403, Loss: 2.4123799800872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1634 , TFLOPS: 100.67753394446105, Tokens per sec: 82266.2046563043, Loss: 2.408082962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1635 , TFLOPS: 102.84341366059641, Tokens per sec: 84036.00072706248, Loss: 2.416268825531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1636 , TFLOPS: 100.69139160003449, Tokens per sec: 82277.52810340116, Loss: 2.410600423812866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1637 , TFLOPS: 102.81616474570151, Tokens per sec: 84013.73493725239, Loss: 2.4287946224212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1638 , TFLOPS: 100.80714249802601, Tokens per sec: 82372.11114184321, Loss: 2.4140443801879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1639 , TFLOPS: 102.00303619124732, Tokens per sec: 83349.30666361672, Loss: 2.4367308616638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1640 , TFLOPS: 101.45687258431346, Tokens per sec: 82903.02232088907, Loss: 2.4069817066192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1641 , TFLOPS: 102.10089799513699, Tokens per sec: 83429.27206276193, Loss: 2.429777145385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1642 , TFLOPS: 100.34162747723241, Tokens per sec: 81991.72683493006, Loss: 2.412651538848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1643 , TFLOPS: 101.72455005103214, Tokens per sec: 83121.7484695759, Loss: 2.414126396179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1644 , TFLOPS: 102.3874757345798, Tokens per sec: 83663.44211082795, Loss: 2.402157783508301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1645 , TFLOPS: 101.73595022983731, Tokens per sec: 83131.06384914431, Loss: 2.419036388397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1646 , TFLOPS: 101.24587476117102, Tokens per sec: 82730.61056803219, Loss: 2.4230570793151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1647 , TFLOPS: 102.94788047245342, Tokens per sec: 84121.36324824582, Loss: 2.4189517498016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1648 , TFLOPS: 103.48698652349995, Tokens per sec: 84561.8806803803, Loss: 2.429642677307129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1649 , TFLOPS: 101.4949821815181, Tokens per sec: 82934.16265379328, Loss: 2.4256742000579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1650 , TFLOPS: 102.73179097879532, Tokens per sec: 83944.79096033935, Loss: 2.4120357036590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1651 , TFLOPS: 103.31956948178501, Tokens per sec: 84425.07990589697, Loss: 2.403872489929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1652 , TFLOPS: 102.70365904518934, Tokens per sec: 83921.80363320952, Loss: 2.4075441360473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1653 , TFLOPS: 101.42530688180541, Tokens per sec: 82877.22917279626, Loss: 2.412656307220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1654 , TFLOPS: 101.92400660898316, Tokens per sec: 83284.72955754625, Loss: 2.4032785892486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1655 , TFLOPS: 101.46285707549552, Tokens per sec: 82907.91240268842, Loss: 2.3956847190856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1656 , TFLOPS: 102.70821517809522, Tokens per sec: 83925.52656669219, Loss: 2.405184745788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1657 , TFLOPS: 102.31327026212678, Tokens per sec: 83602.80690906779, Loss: 2.4110729694366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1658 , TFLOPS: 102.5476012799549, Tokens per sec: 83794.28481595218, Loss: 2.408278703689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1659 , TFLOPS: 101.23540551151991, Tokens per sec: 82722.05587463983, Loss: 2.4053497314453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1660 , TFLOPS: 101.42118606287013, Tokens per sec: 82873.86194556493, Loss: 2.411691904067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1661 , TFLOPS: 101.31639091969937, Tokens per sec: 82788.23113641312, Loss: 2.4069955348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1662 , TFLOPS: 101.12745679625662, Tokens per sec: 82633.84820055103, Loss: 2.39078688621521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1663 , TFLOPS: 102.16384931454671, Tokens per sec: 83480.71120636273, Loss: 2.404132604598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1664 , TFLOPS: 101.54566444465964, Tokens per sec: 82975.57643568376, Loss: 2.4014663696289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1665 , TFLOPS: 100.28474398188042, Tokens per sec: 81945.24586656652, Loss: 2.4060025215148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1666 , TFLOPS: 101.56680657635327, Tokens per sec: 82992.85221573751, Loss: 2.3930530548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1667 , TFLOPS: 102.95342854606969, Tokens per sec: 84125.89672201779, Loss: 2.4278156757354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1668 , TFLOPS: 101.45632250126415, Tokens per sec: 82902.57283386907, Loss: 2.4053444862365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1669 , TFLOPS: 102.09642040160715, Tokens per sec: 83425.61330582517, Loss: 2.394716739654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1670 , TFLOPS: 101.50781159544277, Tokens per sec: 82944.64589816962, Loss: 2.4043378829956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1671 , TFLOPS: 102.23601333615294, Tokens per sec: 83539.67828608435, Loss: 2.397813320159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1672 , TFLOPS: 101.5874596383511, Tokens per sec: 83009.72836435305, Loss: 2.4230294227600098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1673 , TFLOPS: 102.3283698076518, Tokens per sec: 83615.14513641316, Loss: 2.425363540649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1674 , TFLOPS: 100.8768789790313, Tokens per sec: 82429.09461564937, Loss: 2.40670108795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1675 , TFLOPS: 101.46488974773027, Tokens per sec: 82909.57335149708, Loss: 2.387814521789551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1676 , TFLOPS: 101.48836423233527, Tokens per sec: 82928.75495715482, Loss: 2.4013240337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1677 , TFLOPS: 102.16339669037137, Tokens per sec: 83480.34135549748, Loss: 2.4173436164855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1678 , TFLOPS: 101.42334743446139, Tokens per sec: 82875.6280579308, Loss: 2.4113495349884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1679 , TFLOPS: 102.14404923269122, Tokens per sec: 83464.53204977915, Loss: 2.3971502780914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1680 , TFLOPS: 100.35250763942241, Tokens per sec: 82000.6172955358, Loss: 2.3862037658691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1681 , TFLOPS: 100.96460061625977, Tokens per sec: 82500.77422358458, Loss: 2.400721311569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1682 , TFLOPS: 102.22671465731545, Tokens per sec: 83532.08009623697, Loss: 2.3950607776641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1683 , TFLOPS: 102.28241185677618, Tokens per sec: 83577.59171168978, Loss: 2.402608871459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1684 , TFLOPS: 101.10658978893937, Tokens per sec: 82616.79723170756, Loss: 2.4107165336608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1685 , TFLOPS: 101.70989740091098, Tokens per sec: 83109.77541196912, Loss: 2.4092538356781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1686 , TFLOPS: 103.37353868530325, Tokens per sec: 84469.17953138262, Loss: 2.4003095626831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1687 , TFLOPS: 101.40594354297943, Tokens per sec: 82861.40689018487, Loss: 2.3980343341827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1688 , TFLOPS: 102.77546648155403, Tokens per sec: 83980.47933795095, Loss: 2.4003589153289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1689 , TFLOPS: 102.82409257087879, Tokens per sec: 84020.21296728512, Loss: 2.3855714797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1690 , TFLOPS: 102.73250047515387, Tokens per sec: 83945.37070807799, Loss: 2.404320001602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1691 , TFLOPS: 102.21135154310205, Tokens per sec: 83519.52650013145, Loss: 2.4173357486724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1692 , TFLOPS: 102.0247351410621, Tokens per sec: 83367.03743409105, Loss: 2.4162442684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1693 , TFLOPS: 101.48080215684558, Tokens per sec: 82922.57579060706, Loss: 2.3915855884552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1694 , TFLOPS: 102.32051311152652, Tokens per sec: 83608.72522775985, Loss: 2.427520990371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1695 , TFLOPS: 102.67732355751171, Tokens per sec: 83900.28422829235, Loss: 2.4215760231018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1696 , TFLOPS: 101.47116283799542, Tokens per sec: 82914.69925503645, Loss: 2.3879952430725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1697 , TFLOPS: 101.63154562639237, Tokens per sec: 83045.75216005585, Loss: 2.400514602661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1698 , TFLOPS: 101.1792585326054, Tokens per sec: 82676.17673281695, Loss: 2.409832239151001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1699 , TFLOPS: 102.03555043418336, Tokens per sec: 83375.87488851085, Loss: 2.4038774967193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1700 , TFLOPS: 100.96744581407994, Tokens per sec: 82503.09910796535, Loss: 2.3984313011169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1701 , TFLOPS: 102.10468132790733, Tokens per sec: 83432.36352136022, Loss: 2.4209933280944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1702 , TFLOPS: 100.92337585998709, Tokens per sec: 82467.08841401454, Loss: 2.3984556198120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1703 , TFLOPS: 100.68922601545768, Tokens per sec: 82275.75854849633, Loss: 2.406912326812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1704 , TFLOPS: 102.17832688277909, Tokens per sec: 83492.54120005117, Loss: 2.4126811027526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1705 , TFLOPS: 102.17426177987271, Tokens per sec: 83489.21949982131, Loss: 2.3783509731292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1706 , TFLOPS: 101.97925737652217, Tokens per sec: 83329.87638198366, Loss: 2.3915953636169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1707 , TFLOPS: 102.09229719096972, Tokens per sec: 83422.24412427245, Loss: 2.4123544692993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1708 , TFLOPS: 102.16569777049529, Tokens per sec: 83482.2216273018, Loss: 2.3932905197143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1709 , TFLOPS: 102.7844691207524, Tokens per sec: 83987.8356261885, Loss: 2.401519536972046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1710 , TFLOPS: 102.21665560512368, Tokens per sec: 83523.86058574801, Loss: 2.4092860221862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1711 , TFLOPS: 102.03892614260218, Tokens per sec: 83378.63326675832, Loss: 2.376108407974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1712 , TFLOPS: 100.92536017085204, Tokens per sec: 82468.70984550299, Loss: 2.41650128364563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1713 , TFLOPS: 101.61943474210908, Tokens per sec: 83035.85604474609, Loss: 2.404695510864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1714 , TFLOPS: 100.7473396814479, Tokens per sec: 82323.2447209558, Loss: 2.397784471511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1715 , TFLOPS: 102.00117591343958, Tokens per sec: 83347.78658273222, Loss: 2.4273743629455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1716 , TFLOPS: 102.12718567261307, Tokens per sec: 83450.75240073264, Loss: 2.3910536766052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1717 , TFLOPS: 101.45785410034216, Tokens per sec: 82903.82434290258, Loss: 2.3923544883728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1718 , TFLOPS: 100.89735305216999, Tokens per sec: 82445.82450786053, Loss: 2.396253824234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1719 , TFLOPS: 100.46453201803354, Tokens per sec: 82092.155299063, Loss: 2.4244110584259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1720 , TFLOPS: 102.91419910948015, Tokens per sec: 84093.84133952495, Loss: 2.3994839191436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1721 , TFLOPS: 101.54771976088415, Tokens per sec: 82977.25588747935, Loss: 2.4167301654815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1722 , TFLOPS: 101.79947755582471, Tokens per sec: 83182.97366254748, Loss: 2.395124673843384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1723 , TFLOPS: 101.32303220769376, Tokens per sec: 82793.6579037953, Loss: 2.4016802310943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1724 , TFLOPS: 103.3028637045121, Tokens per sec: 84411.4291852423, Loss: 2.4038243293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1725 , TFLOPS: 101.51127230058337, Tokens per sec: 82947.47372942648, Loss: 2.3810627460479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1726 , TFLOPS: 102.74830612416045, Tokens per sec: 83958.28591075486, Loss: 2.4109907150268555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1727 , TFLOPS: 102.27073717195047, Tokens per sec: 83568.05202618559, Loss: 2.395211935043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1728 , TFLOPS: 102.14722498638804, Tokens per sec: 83467.127039876, Loss: 2.4272584915161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1729 , TFLOPS: 102.14421017596139, Tokens per sec: 83464.66356066827, Loss: 2.38427472114563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1730 , TFLOPS: 102.08139116686002, Tokens per sec: 83413.33253122636, Loss: 2.40048885345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1731 , TFLOPS: 101.30584845673906, Tokens per sec: 82779.61661854111, Loss: 2.399698257446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1732 , TFLOPS: 102.07034243827968, Tokens per sec: 83404.30433068381, Loss: 2.3887434005737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1733 , TFLOPS: 102.71831123277187, Tokens per sec: 83933.77631286405, Loss: 2.4222309589385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1734 , TFLOPS: 102.15882891993768, Tokens per sec: 83476.60891269124, Loss: 2.408479690551758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1735 , TFLOPS: 101.63880906852897, Tokens per sec: 83051.68730560331, Loss: 2.371199369430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1736 , TFLOPS: 101.48306318089486, Tokens per sec: 82924.42333156166, Loss: 2.3947479724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1737 , TFLOPS: 100.62594501211014, Tokens per sec: 82224.05001166312, Loss: 2.3940422534942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1738 , TFLOPS: 100.9589194490246, Tokens per sec: 82496.13199559048, Loss: 2.372896194458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1739 , TFLOPS: 102.18105126586958, Tokens per sec: 83494.76736360628, Loss: 2.4031457901000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1740 , TFLOPS: 101.62805243948767, Tokens per sec: 83042.8977871134, Loss: 2.404552459716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1741 , TFLOPS: 100.62524258284256, Tokens per sec: 82223.47603862644, Loss: 2.4040961265563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1742 , TFLOPS: 102.12977666428438, Tokens per sec: 83452.86956672533, Loss: 2.4213011264801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1743 , TFLOPS: 102.33653602980291, Tokens per sec: 83621.8179667501, Loss: 2.404080629348755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1744 , TFLOPS: 101.36941194622888, Tokens per sec: 82831.55598207303, Loss: 2.3847744464874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1745 , TFLOPS: 102.7732485848161, Tokens per sec: 83978.66703743265, Loss: 2.394233465194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1746 , TFLOPS: 102.25944891203088, Tokens per sec: 83558.82809841962, Loss: 2.3793747425079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1747 , TFLOPS: 102.72269533939298, Tokens per sec: 83937.35867924112, Loss: 2.416504383087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1748 , TFLOPS: 101.47039491761987, Tokens per sec: 82914.07176752955, Loss: 2.3883819580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1749 , TFLOPS: 102.83393184688487, Tokens per sec: 84028.25289299517, Loss: 2.3977460861206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1750 , TFLOPS: 101.04383216636087, Tokens per sec: 82565.51636277373, Loss: 2.4061334133148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1751 , TFLOPS: 101.46435548549009, Tokens per sec: 82909.1367920675, Loss: 2.410162925720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1752 , TFLOPS: 100.9255114474307, Tokens per sec: 82468.83345748961, Loss: 2.388918399810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1753 , TFLOPS: 102.0871843673581, Tokens per sec: 83418.06630448361, Loss: 2.3976407051086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1754 , TFLOPS: 102.16682824220372, Tokens per sec: 83483.1453648357, Loss: 2.418849468231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1755 , TFLOPS: 98.65048718885052, Tokens per sec: 80609.85257146931, Loss: 2.414452075958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1756 , TFLOPS: 100.90702759633135, Tokens per sec: 82453.72982693969, Loss: 2.3986001014709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1757 , TFLOPS: 101.76108539263788, Tokens per sec: 83151.6024377053, Loss: 2.3870251178741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1758 , TFLOPS: 102.1294799184995, Tokens per sec: 83452.6270881056, Loss: 2.4194765090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1759 , TFLOPS: 101.77369587013538, Tokens per sec: 83161.90678349108, Loss: 2.4203572273254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1760 , TFLOPS: 101.68619228940335, Tokens per sec: 83090.40535513239, Loss: 2.393627166748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1761 , TFLOPS: 101.82904171798461, Tokens per sec: 83207.13130049755, Loss: 2.4049277305603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1762 , TFLOPS: 102.6795989146815, Tokens per sec: 83902.1434812087, Loss: 2.398822784423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1763 , TFLOPS: 101.4600579518432, Tokens per sec: 82905.62516669626, Loss: 2.4166173934936523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1764 , TFLOPS: 102.04858932849336, Tokens per sec: 83386.52930470253, Loss: 2.403001546859741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1765 , TFLOPS: 102.70631787577867, Tokens per sec: 83923.97623212842, Loss: 2.3950796127319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1766 , TFLOPS: 102.76499710235106, Tokens per sec: 83971.92453869841, Loss: 2.398205518722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1767 , TFLOPS: 100.83033607826452, Tokens per sec: 82391.06321331198, Loss: 2.398735284805298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1768 , TFLOPS: 102.2069050105731, Tokens per sec: 83515.8931239381, Loss: 2.392112970352173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1769 , TFLOPS: 100.39141626332025, Tokens per sec: 82032.41053371981, Loss: 2.4188013076782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1770 , TFLOPS: 101.39887493762737, Tokens per sec: 82855.63095078962, Loss: 2.4161229133605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1771 , TFLOPS: 102.86145123495102, Tokens per sec: 84050.73969339608, Loss: 2.3997626304626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1772 , TFLOPS: 102.12888700354048, Tokens per sec: 83452.14260202959, Loss: 2.3908588886260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1773 , TFLOPS: 101.603579052819, Tokens per sec: 83022.89995286541, Loss: 2.3908636569976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1774 , TFLOPS: 101.72852698941816, Tokens per sec: 83124.99812830662, Loss: 2.3971405029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1775 , TFLOPS: 101.56555393199778, Tokens per sec: 82991.8286477887, Loss: 2.396908760070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1776 , TFLOPS: 101.51590602697912, Tokens per sec: 82951.26006654718, Loss: 2.406794786453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1777 , TFLOPS: 102.15218501473416, Tokens per sec: 83471.18000672011, Loss: 2.384052276611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1778 , TFLOPS: 101.44890922962038, Tokens per sec: 82896.51525877437, Loss: 2.3923261165618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1779 , TFLOPS: 101.0735006884647, Tokens per sec: 82589.75927592049, Loss: 2.394383668899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1780 , TFLOPS: 102.0475668393345, Tokens per sec: 83385.69380248961, Loss: 2.3989760875701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1781 , TFLOPS: 102.85713111020459, Tokens per sec: 84047.20960825589, Loss: 2.4078385829925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1782 , TFLOPS: 102.15434376133791, Tokens per sec: 83472.94397414112, Loss: 2.400482654571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1783 , TFLOPS: 101.27567620996531, Tokens per sec: 82754.96209899956, Loss: 2.399815559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1784 , TFLOPS: 102.18329265684807, Tokens per sec: 83496.59886187332, Loss: 2.4104228019714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1785 , TFLOPS: 102.79258486897835, Tokens per sec: 83994.46721298186, Loss: 2.4049272537231445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1786 , TFLOPS: 102.3130786458006, Tokens per sec: 83602.65033443502, Loss: 2.389751434326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1787 , TFLOPS: 102.08442187788393, Tokens per sec: 83415.80900322145, Loss: 2.426710844039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1788 , TFLOPS: 100.35305235358295, Tokens per sec: 82001.06239549856, Loss: 2.3870277404785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1789 , TFLOPS: 102.00091996627215, Tokens per sec: 83347.57744171305, Loss: 2.406562089920044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1790 , TFLOPS: 100.79939819918911, Tokens per sec: 82365.7830759076, Loss: 2.4037344455718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1791 , TFLOPS: 101.66459323379644, Tokens per sec: 83072.75621078681, Loss: 2.3906497955322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1792 , TFLOPS: 102.2401125528129, Tokens per sec: 83543.02786154044, Loss: 2.411041259765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1793 , TFLOPS: 101.46369583415691, Tokens per sec: 82908.59777398223, Loss: 2.4197163581848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1794 , TFLOPS: 100.29846697651207, Tokens per sec: 81956.45927874134, Loss: 2.4000723361968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1795 , TFLOPS: 101.14153534040722, Tokens per sec: 82645.35214139047, Loss: 2.400907278060913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1796 , TFLOPS: 102.36809377757692, Tokens per sec: 83647.60461482493, Loss: 2.39192533493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1797 , TFLOPS: 101.69328784617885, Tokens per sec: 83096.20331722952, Loss: 2.389957904815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1798 , TFLOPS: 101.60099445202378, Tokens per sec: 83020.78800902209, Loss: 2.396580457687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1799 , TFLOPS: 101.5191946832043, Tokens per sec: 82953.94731220632, Loss: 2.4011590480804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1800 , TFLOPS: 103.40245262094709, Tokens per sec: 84492.8058525081, Loss: 2.3916828632354736 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/model.pt +[rank0]:[2024-08-27 06:13:24,838] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007783559995004907, 'preprocessing_with_comm': 0.001524393999716267, 'state_converting': 3.3336062809976283, : 3.3445236530096736}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800, took 15.70s + eval ppl=8.314668655395508, eval loss=2.118021249771118 +------------------------------------------------------------------ +iteration: 1801 , TFLOPS: 100.820486299099, Tokens per sec: 82383.01470519997, Loss: 2.375997304916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1802 , TFLOPS: 99.56333961993688, Tokens per sec: 81355.76779182209, Loss: 2.413283348083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1803 , TFLOPS: 102.29507413073044, Tokens per sec: 83587.93838169375, Loss: 2.374866008758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1804 , TFLOPS: 100.89251807958718, Tokens per sec: 82441.87372729983, Loss: 2.4129724502563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1805 , TFLOPS: 101.12109932556687, Tokens per sec: 82628.65334759459, Loss: 2.3880205154418945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1806 , TFLOPS: 102.69468767932362, Tokens per sec: 83914.47289921713, Loss: 2.3866753578186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1807 , TFLOPS: 101.31024596602414, Tokens per sec: 82783.20993657991, Loss: 2.408317804336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1808 , TFLOPS: 101.63001720579393, Tokens per sec: 83044.50324823982, Loss: 2.3865528106689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1809 , TFLOPS: 102.53827099612816, Tokens per sec: 83786.66080085449, Loss: 2.4208037853240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1810 , TFLOPS: 102.76715328762437, Tokens per sec: 83973.68641319037, Loss: 2.39205265045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1811 , TFLOPS: 101.90538932522696, Tokens per sec: 83269.51689573753, Loss: 2.4041755199432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1812 , TFLOPS: 101.92464967831704, Tokens per sec: 83285.25502605304, Loss: 2.4005045890808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1813 , TFLOPS: 103.36102327988192, Tokens per sec: 84458.95287143764, Loss: 2.395977258682251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1814 , TFLOPS: 101.54862085331104, Tokens per sec: 82977.99219329753, Loss: 2.3905906677246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1815 , TFLOPS: 103.44072030337252, Tokens per sec: 84524.07536091571, Loss: 2.3865723609924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1816 , TFLOPS: 100.62496125068587, Tokens per sec: 82223.24615487907, Loss: 2.3975186347961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1817 , TFLOPS: 100.69804291205267, Tokens per sec: 82282.96306167121, Loss: 2.376279592514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1818 , TFLOPS: 102.68646312305374, Tokens per sec: 83907.7524025701, Loss: 2.3872318267822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1819 , TFLOPS: 102.18947582582214, Tokens per sec: 83501.65128841098, Loss: 2.4043703079223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1820 , TFLOPS: 101.95268489759148, Tokens per sec: 83308.16332541294, Loss: 2.4045751094818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1821 , TFLOPS: 102.09945377097394, Tokens per sec: 83428.09195002065, Loss: 2.3755123615264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1822 , TFLOPS: 101.55637383440265, Tokens per sec: 82984.32735372697, Loss: 2.3865554332733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1823 , TFLOPS: 102.5902670157598, Tokens per sec: 83829.14808699212, Loss: 2.4202351570129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1824 , TFLOPS: 101.71708372178878, Tokens per sec: 83115.64753975073, Loss: 2.371410369873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1825 , TFLOPS: 102.62623502137657, Tokens per sec: 83858.53847028043, Loss: 2.4018421173095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1826 , TFLOPS: 102.76803376508838, Tokens per sec: 83974.40587399172, Loss: 2.4058847427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1827 , TFLOPS: 100.94747874714004, Tokens per sec: 82486.78350357072, Loss: 2.4044175148010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1828 , TFLOPS: 102.76710556764739, Tokens per sec: 83973.64741996877, Loss: 2.3833048343658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1829 , TFLOPS: 102.7483756102447, Tokens per sec: 83958.34268962302, Loss: 2.3918566703796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1830 , TFLOPS: 99.86519524179484, Tokens per sec: 81602.42179089719, Loss: 2.4060122966766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1831 , TFLOPS: 103.31256338430346, Tokens per sec: 84419.3550432918, Loss: 2.3815736770629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1832 , TFLOPS: 102.39616938773557, Tokens per sec: 83670.54591861606, Loss: 2.3926730155944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1833 , TFLOPS: 100.14785407171122, Tokens per sec: 81833.38959710748, Loss: 2.399932384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1834 , TFLOPS: 102.19752919005414, Tokens per sec: 83508.23189963697, Loss: 2.3988819122314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1835 , TFLOPS: 101.2826942729449, Tokens per sec: 82760.6967389212, Loss: 2.390676498413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1836 , TFLOPS: 100.69850857615256, Tokens per sec: 82283.3435677945, Loss: 2.4056692123413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1837 , TFLOPS: 102.19003446658483, Tokens per sec: 83502.10776815886, Loss: 2.3850998878479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1838 , TFLOPS: 101.06691263152649, Tokens per sec: 82584.3760050046, Loss: 2.402656078338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1839 , TFLOPS: 100.45804193622023, Tokens per sec: 82086.85208614377, Loss: 2.4085967540740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1840 , TFLOPS: 100.50447335616873, Tokens per sec: 82124.79239463479, Loss: 2.362102508544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1841 , TFLOPS: 102.24924371443579, Tokens per sec: 83550.48916875951, Loss: 2.4147329330444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1842 , TFLOPS: 100.81836333920825, Tokens per sec: 82381.27998002338, Loss: 2.390200138092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1843 , TFLOPS: 101.18047583715355, Tokens per sec: 82677.17142369946, Loss: 2.3829002380371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1844 , TFLOPS: 102.65718551490346, Tokens per sec: 83883.82891527782, Loss: 2.394735097885132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1845 , TFLOPS: 101.40507645816405, Tokens per sec: 82860.69837286147, Loss: 2.404589891433716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1846 , TFLOPS: 101.69810403869914, Tokens per sec: 83100.13875211768, Loss: 2.3829288482666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1847 , TFLOPS: 102.6609789032129, Tokens per sec: 83886.92859051595, Loss: 2.404815196990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1848 , TFLOPS: 102.66750303979013, Tokens per sec: 83892.25963045945, Loss: 2.399637460708618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1849 , TFLOPS: 101.97299786462116, Tokens per sec: 83324.76157367519, Loss: 2.379171848297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1850 , TFLOPS: 102.70822196488551, Tokens per sec: 83925.53211235325, Loss: 2.3908801078796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1851 , TFLOPS: 102.51785238194164, Tokens per sec: 83769.97622558079, Loss: 2.391155242919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1852 , TFLOPS: 102.0360952611736, Tokens per sec: 83376.32008066965, Loss: 2.382242441177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1853 , TFLOPS: 102.63562934552577, Tokens per sec: 83866.21481437414, Loss: 2.3811566829681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1854 , TFLOPS: 100.9589638986236, Tokens per sec: 82496.16831650204, Loss: 2.405435800552368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1855 , TFLOPS: 100.48895141549518, Tokens per sec: 82112.10901734013, Loss: 2.414616346359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1856 , TFLOPS: 102.76358429191035, Tokens per sec: 83970.77009491833, Loss: 2.3766167163848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1857 , TFLOPS: 102.18403241281652, Tokens per sec: 83497.20333551816, Loss: 2.3738820552825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1858 , TFLOPS: 102.18383047211786, Tokens per sec: 83497.03832457488, Loss: 2.3967368602752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1859 , TFLOPS: 102.24244800869893, Tokens per sec: 83544.9362226649, Loss: 2.405893087387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1860 , TFLOPS: 101.40742850698287, Tokens per sec: 82862.62029249845, Loss: 2.395031213760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1861 , TFLOPS: 102.6611390246865, Tokens per sec: 83887.05942989394, Loss: 2.3794426918029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1862 , TFLOPS: 101.46358961724484, Tokens per sec: 82908.5109814091, Loss: 2.375309705734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1863 , TFLOPS: 101.97699649449162, Tokens per sec: 83328.02895707623, Loss: 2.4012372493743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1864 , TFLOPS: 102.71459907295613, Tokens per sec: 83930.74301151912, Loss: 2.3803863525390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1865 , TFLOPS: 100.43475512064957, Tokens per sec: 82067.82382968502, Loss: 2.3859026432037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1866 , TFLOPS: 102.0824989464608, Tokens per sec: 83414.23772645497, Loss: 2.3962368965148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1867 , TFLOPS: 103.4230462161261, Tokens per sec: 84509.63340925518, Loss: 2.4035892486572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1868 , TFLOPS: 99.9426599192143, Tokens per sec: 81665.72017293483, Loss: 2.3724446296691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1869 , TFLOPS: 103.42387397457132, Tokens per sec: 84510.30979198916, Loss: 2.403000593185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1870 , TFLOPS: 102.23734343419098, Tokens per sec: 83540.76514343101, Loss: 2.3853020668029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1871 , TFLOPS: 99.92610769939103, Tokens per sec: 81652.1949280252, Loss: 2.3881447315216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1872 , TFLOPS: 102.05892319617178, Tokens per sec: 83394.973374, Loss: 2.3968276977539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1873 , TFLOPS: 101.47842374885964, Tokens per sec: 82920.63233221608, Loss: 2.3859119415283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1874 , TFLOPS: 100.49566218123499, Tokens per sec: 82117.5925567766, Loss: 2.4009921550750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1875 , TFLOPS: 102.21559163955271, Tokens per sec: 83522.99119209108, Loss: 2.4002811908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1876 , TFLOPS: 101.0410939344864, Tokens per sec: 82563.27888302074, Loss: 2.3817169666290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1877 , TFLOPS: 101.0698176636502, Tokens per sec: 82586.74978153507, Loss: 2.3904662132263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1878 , TFLOPS: 99.8535330698278, Tokens per sec: 81592.89232997192, Loss: 2.3837835788726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1879 , TFLOPS: 102.11767165182215, Tokens per sec: 83442.97824943172, Loss: 2.39921498298645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1880 , TFLOPS: 100.82027311908502, Tokens per sec: 82382.84051032279, Loss: 2.395811080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1881 , TFLOPS: 101.05131630118457, Tokens per sec: 82571.63184199695, Loss: 2.390619993209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1882 , TFLOPS: 102.00090703905775, Tokens per sec: 83347.56687855338, Loss: 2.3904995918273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1883 , TFLOPS: 101.80071794337631, Tokens per sec: 83183.98721514629, Loss: 2.3744280338287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1884 , TFLOPS: 101.15041705213737, Tokens per sec: 82652.60961668077, Loss: 2.4211535453796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1885 , TFLOPS: 102.66055436125863, Tokens per sec: 83886.58168635645, Loss: 2.4144115447998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1886 , TFLOPS: 102.60576364607114, Tokens per sec: 83841.81078253828, Loss: 2.3990819454193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1887 , TFLOPS: 101.45565674248652, Tokens per sec: 82902.02882523372, Loss: 2.395292282104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1888 , TFLOPS: 102.55898119651685, Tokens per sec: 83803.58363872007, Loss: 2.387927293777466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1889 , TFLOPS: 102.91089252958133, Tokens per sec: 84091.13944797068, Loss: 2.3778488636016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1890 , TFLOPS: 101.47724476163127, Tokens per sec: 82919.66895140194, Loss: 2.395822048187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1891 , TFLOPS: 103.38177939985482, Tokens per sec: 84475.91323137758, Loss: 2.3690829277038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1892 , TFLOPS: 100.78448640285211, Tokens per sec: 82353.59826325685, Loss: 2.3846023082733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1893 , TFLOPS: 101.99904794691562, Tokens per sec: 83346.04776650676, Loss: 2.3926167488098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1894 , TFLOPS: 102.68407313149693, Tokens per sec: 83905.79947894516, Loss: 2.3953471183776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1895 , TFLOPS: 102.64322572306871, Tokens per sec: 83872.42201001263, Loss: 2.3802826404571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1896 , TFLOPS: 102.05366783899049, Tokens per sec: 83390.67908636224, Loss: 2.3813281059265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1897 , TFLOPS: 102.2414080829306, Tokens per sec: 83544.08647254952, Loss: 2.375628709793091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1898 , TFLOPS: 102.34164571991187, Tokens per sec: 83625.99322607325, Loss: 2.380481719970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1899 , TFLOPS: 102.07346363814669, Tokens per sec: 83406.85474344197, Loss: 2.4033584594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1900 , TFLOPS: 101.09829892587301, Tokens per sec: 82610.02255407022, Loss: 2.362699031829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1901 , TFLOPS: 102.69609496361724, Tokens per sec: 83915.62282744017, Loss: 2.400134325027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1902 , TFLOPS: 102.7597171698369, Tokens per sec: 83967.61016992354, Loss: 2.4119231700897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1903 , TFLOPS: 100.86177837078596, Tokens per sec: 82416.75551992786, Loss: 2.393536329269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1904 , TFLOPS: 102.09346918832772, Tokens per sec: 83423.20179348357, Loss: 2.410770893096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1905 , TFLOPS: 102.63867025599859, Tokens per sec: 83868.69962060147, Loss: 2.407184362411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1906 , TFLOPS: 100.56889557317331, Tokens per sec: 82177.43344652462, Loss: 2.377091884613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1907 , TFLOPS: 102.77423315880296, Tokens per sec: 83979.4715581825, Loss: 2.405181407928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1908 , TFLOPS: 102.28018059906383, Tokens per sec: 83575.76849357534, Loss: 2.3847503662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1909 , TFLOPS: 99.6850853681292, Tokens per sec: 81455.24937668438, Loss: 2.401625394821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1910 , TFLOPS: 102.25682377638886, Tokens per sec: 83556.68303250952, Loss: 2.382981061935425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1911 , TFLOPS: 101.56958661859335, Tokens per sec: 82995.12385981652, Loss: 2.3945813179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1912 , TFLOPS: 100.87581273838576, Tokens per sec: 82428.22336297043, Loss: 2.407926559448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1913 , TFLOPS: 101.64273541014776, Tokens per sec: 83054.89562041305, Loss: 2.3881473541259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1914 , TFLOPS: 101.29156731105448, Tokens per sec: 82767.94712677285, Loss: 2.3795127868652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1915 , TFLOPS: 100.38421533673264, Tokens per sec: 82026.52647123672, Loss: 2.3948333263397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1916 , TFLOPS: 100.59320602039075, Tokens per sec: 82197.29813875258, Loss: 2.3840785026550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1917 , TFLOPS: 102.16509532111458, Tokens per sec: 83481.72935040439, Loss: 2.3871631622314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1918 , TFLOPS: 101.35698171695701, Tokens per sec: 82821.3989217524, Loss: 2.4186184406280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1919 , TFLOPS: 101.74663687420953, Tokens per sec: 83139.79618135955, Loss: 2.384598970413208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1920 , TFLOPS: 102.05038484692824, Tokens per sec: 83387.99646903643, Loss: 2.398483991622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1921 , TFLOPS: 102.4469873745375, Tokens per sec: 83712.07060380328, Loss: 2.388976812362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1922 , TFLOPS: 101.63389576360008, Tokens per sec: 83047.67251766619, Loss: 2.4019320011138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1923 , TFLOPS: 102.78886009401464, Tokens per sec: 83991.42360347477, Loss: 2.3853461742401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1924 , TFLOPS: 101.43066449640388, Tokens per sec: 82881.60701760194, Loss: 2.4202821254730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1925 , TFLOPS: 102.21546539487707, Tokens per sec: 83522.88803431678, Loss: 2.373039960861206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1926 , TFLOPS: 103.30278030016399, Tokens per sec: 84411.36103340244, Loss: 2.4052014350891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1927 , TFLOPS: 102.2008988347796, Tokens per sec: 83510.98532308452, Loss: 2.392005205154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1928 , TFLOPS: 100.90999610400297, Tokens per sec: 82456.15547097435, Loss: 2.3904306888580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1929 , TFLOPS: 102.69898654904507, Tokens per sec: 83917.98561632927, Loss: 2.4027791023254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1930 , TFLOPS: 102.77106376605798, Tokens per sec: 83976.88176578315, Loss: 2.398239850997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1931 , TFLOPS: 101.97698471319518, Tokens per sec: 83328.01933027561, Loss: 2.3788363933563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1932 , TFLOPS: 101.62239654127252, Tokens per sec: 83038.27620708603, Loss: 2.367994546890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1933 , TFLOPS: 102.74964807334328, Tokens per sec: 83959.38245197698, Loss: 2.363003969192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1934 , TFLOPS: 101.93849701311623, Tokens per sec: 83296.57003977957, Loss: 2.400840997695923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1935 , TFLOPS: 101.63188466474912, Tokens per sec: 83046.02919702478, Loss: 2.3827195167541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1936 , TFLOPS: 102.6274992132288, Tokens per sec: 83859.57147398607, Loss: 2.376885414123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1937 , TFLOPS: 100.82781317590258, Tokens per sec: 82389.00168484673, Loss: 2.403127908706665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1938 , TFLOPS: 101.68393552216689, Tokens per sec: 83088.56129252912, Loss: 2.382944345474243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1939 , TFLOPS: 101.75048612754824, Tokens per sec: 83142.9415053512, Loss: 2.4163689613342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1940 , TFLOPS: 102.75870594336048, Tokens per sec: 83966.78387073838, Loss: 2.3814148902893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1941 , TFLOPS: 100.12194427071287, Tokens per sec: 81812.21803173411, Loss: 2.3551580905914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1942 , TFLOPS: 101.55491811807386, Tokens per sec: 82983.13785044126, Loss: 2.3602242469787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1943 , TFLOPS: 101.45185790443212, Tokens per sec: 82898.92469686866, Loss: 2.3939714431762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1944 , TFLOPS: 101.5375882287392, Tokens per sec: 82968.97715175487, Loss: 2.3805389404296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1945 , TFLOPS: 101.70665110042505, Tokens per sec: 83107.12277627486, Loss: 2.39518404006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1946 , TFLOPS: 102.24848870214349, Tokens per sec: 83549.87222877973, Loss: 2.410139799118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1947 , TFLOPS: 100.49004368218225, Tokens per sec: 82113.00153656758, Loss: 2.4014956951141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1948 , TFLOPS: 101.6237571403514, Tokens per sec: 83039.38798761832, Loss: 2.3971405029296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1949 , TFLOPS: 102.22811180054047, Tokens per sec: 83533.22173793179, Loss: 2.3979640007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1950 , TFLOPS: 100.7750884101499, Tokens per sec: 82345.91892149403, Loss: 2.3774490356445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1951 , TFLOPS: 102.30929119695432, Tokens per sec: 83599.55551248511, Loss: 2.389533042907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1952 , TFLOPS: 101.82709383359334, Tokens per sec: 83205.53963401844, Loss: 2.416907548904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1953 , TFLOPS: 101.62407551216197, Tokens per sec: 83039.64813742036, Loss: 2.3977158069610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1954 , TFLOPS: 101.02485346969586, Tokens per sec: 82550.00838117364, Loss: 2.384082555770874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1955 , TFLOPS: 102.72840805016958, Tokens per sec: 83942.02668227493, Loss: 2.3760228157043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1956 , TFLOPS: 100.19986734805883, Tokens per sec: 81875.89098415195, Loss: 2.405388355255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1957 , TFLOPS: 102.19256914771071, Tokens per sec: 83504.17892135531, Loss: 2.3813412189483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1958 , TFLOPS: 102.10412634696917, Tokens per sec: 83431.91003214917, Loss: 2.412872552871704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1959 , TFLOPS: 102.944794316156, Tokens per sec: 84118.84147049047, Loss: 2.407133102416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1960 , TFLOPS: 102.19030327886101, Tokens per sec: 83502.32742158958, Loss: 2.3799431324005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1961 , TFLOPS: 102.67137496301422, Tokens per sec: 83895.42347859801, Loss: 2.375999689102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1962 , TFLOPS: 100.56875246281453, Tokens per sec: 82177.31650736647, Loss: 2.376843214035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1963 , TFLOPS: 101.98935377419777, Tokens per sec: 83338.1264084288, Loss: 2.3955366611480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1964 , TFLOPS: 102.68331286605432, Tokens per sec: 83905.17824648104, Loss: 2.4027340412139893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1965 , TFLOPS: 102.85933020529511, Tokens per sec: 84049.0065454641, Loss: 2.398420810699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1966 , TFLOPS: 100.38035347828107, Tokens per sec: 82023.37084728286, Loss: 2.3881988525390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1967 , TFLOPS: 101.88915251992191, Tokens per sec: 83256.24938415023, Loss: 2.3746490478515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1968 , TFLOPS: 103.32643033054211, Tokens per sec: 84430.68608203034, Loss: 2.3981077671051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1969 , TFLOPS: 102.1062965751819, Tokens per sec: 83433.68338148844, Loss: 2.382443428039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1970 , TFLOPS: 101.01872979494837, Tokens per sec: 82545.00456890007, Loss: 2.385052442550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1971 , TFLOPS: 101.87727860750314, Tokens per sec: 83246.54690465116, Loss: 2.373185634613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1972 , TFLOPS: 103.40993991000757, Tokens per sec: 84498.9239091394, Loss: 2.4007914066314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1973 , TFLOPS: 100.40303879835636, Tokens per sec: 82041.90760628849, Loss: 2.4026365280151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1974 , TFLOPS: 102.8194420973928, Tokens per sec: 84016.41294568586, Loss: 2.38362717628479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1975 , TFLOPS: 100.0039756225708, Tokens per sec: 81715.82281255408, Loss: 2.394629955291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1976 , TFLOPS: 101.43295109697391, Tokens per sec: 82883.47545779005, Loss: 2.4006505012512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1977 , TFLOPS: 102.20788949832419, Tokens per sec: 83516.69757422249, Loss: 2.397660732269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1978 , TFLOPS: 102.2434881639618, Tokens per sec: 83545.78616030616, Loss: 2.382199764251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1979 , TFLOPS: 100.9472348642196, Tokens per sec: 82486.58422055833, Loss: 2.3699495792388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1980 , TFLOPS: 101.55351172142946, Tokens per sec: 82981.98864753924, Loss: 2.384992837905884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1981 , TFLOPS: 101.48581480428544, Tokens per sec: 82926.6717538672, Loss: 2.3857860565185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1982 , TFLOPS: 101.53039097985142, Tokens per sec: 82963.09609441498, Loss: 2.3871877193450928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1983 , TFLOPS: 101.05916815657604, Tokens per sec: 82578.04779516206, Loss: 2.3958117961883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1984 , TFLOPS: 102.3437471034925, Tokens per sec: 83627.71032069142, Loss: 2.405989646911621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1985 , TFLOPS: 99.23276718997825, Tokens per sec: 81085.64854961149, Loss: 2.4028947353363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1986 , TFLOPS: 100.8467939829712, Tokens per sec: 82404.51139091197, Loss: 2.408076047897339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1987 , TFLOPS: 102.79037404442666, Tokens per sec: 83992.66069132899, Loss: 2.3898868560791016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1988 , TFLOPS: 102.18460652365494, Tokens per sec: 83497.67245626316, Loss: 2.3716938495635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1989 , TFLOPS: 102.8574244957628, Tokens per sec: 84047.44934114792, Loss: 2.37233829498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1990 , TFLOPS: 102.95035646045764, Tokens per sec: 84123.38644178167, Loss: 2.371047258377075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1991 , TFLOPS: 101.3228053816355, Tokens per sec: 82793.47255838408, Loss: 2.3760037422180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1992 , TFLOPS: 102.16249902043783, Tokens per sec: 83479.60784628672, Loss: 2.3800981044769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1993 , TFLOPS: 103.28527707976988, Tokens per sec: 84397.05870144544, Loss: 2.389920234680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1994 , TFLOPS: 99.41500776958377, Tokens per sec: 81234.56201849702, Loss: 2.3551177978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1995 , TFLOPS: 101.60600744560344, Tokens per sec: 83024.88425512196, Loss: 2.3852052688598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1996 , TFLOPS: 102.14670445857114, Tokens per sec: 83466.70170319724, Loss: 2.374397039413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1997 , TFLOPS: 102.89366802930496, Tokens per sec: 84077.06486539678, Loss: 2.3954410552978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1998 , TFLOPS: 100.99471986365499, Tokens per sec: 82525.38543596963, Loss: 2.378024101257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 1999 , TFLOPS: 103.27232306948352, Tokens per sec: 84386.47364617456, Loss: 2.399617910385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2000 , TFLOPS: 101.41757739642094, Tokens per sec: 82870.91321131396, Loss: 2.401374578475952 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/model.pt +[rank0]:[2024-08-27 07:06:20,279] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007667751007829793, 'preprocessing_with_comm': 0.0016614979977020994, 'state_converting': 3.3319098130014027, : 3.342863996003871}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000, took 15.94s + eval ppl=9.074803352355957, eval loss=2.2055017948150635 +------------------------------------------------------------------ +iteration: 2001 , TFLOPS: 101.46188566976494, Tokens per sec: 82907.11864205977, Loss: 2.395524501800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2002 , TFLOPS: 97.45273574342694, Tokens per sec: 79631.13903254854, Loss: 2.387523651123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2003 , TFLOPS: 101.23542579010528, Tokens per sec: 82722.07244479394, Loss: 2.4053847789764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2004 , TFLOPS: 101.46568960629155, Tokens per sec: 82910.22693651769, Loss: 2.3846030235290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2005 , TFLOPS: 101.16102092377973, Tokens per sec: 82661.27431316768, Loss: 2.381585121154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2006 , TFLOPS: 102.21531333044703, Tokens per sec: 83522.76377855652, Loss: 2.382373809814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2007 , TFLOPS: 102.93631055747879, Tokens per sec: 84111.90917288394, Loss: 2.391899824142456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2008 , TFLOPS: 101.69373600528262, Tokens per sec: 83096.56951956986, Loss: 2.380772352218628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2009 , TFLOPS: 103.34726131571723, Tokens per sec: 84447.7076162542, Loss: 2.367957830429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2010 , TFLOPS: 102.56249674509158, Tokens per sec: 83806.45628395876, Loss: 2.3817787170410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2011 , TFLOPS: 101.97880575169485, Tokens per sec: 83329.5073477114, Loss: 2.3739004135131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2012 , TFLOPS: 102.7045078210566, Tokens per sec: 83922.49718982007, Loss: 2.3688158988952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2013 , TFLOPS: 101.18305435841314, Tokens per sec: 82679.2783997977, Loss: 2.3945696353912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2014 , TFLOPS: 102.36463182907126, Tokens per sec: 83644.77576758247, Loss: 2.3995909690856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2015 , TFLOPS: 102.90178761153588, Tokens per sec: 84083.6995850543, Loss: 2.3863525390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2016 , TFLOPS: 102.65495928531226, Tokens per sec: 83882.00980577059, Loss: 2.3887290954589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2017 , TFLOPS: 101.42445045999855, Tokens per sec: 82876.52936849163, Loss: 2.3718929290771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2018 , TFLOPS: 103.40899581853265, Tokens per sec: 84498.15246769215, Loss: 2.407778263092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2019 , TFLOPS: 102.21264779548912, Tokens per sec: 83520.58570132543, Loss: 2.402052402496338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2020 , TFLOPS: 102.47769194168399, Tokens per sec: 83737.16009602456, Loss: 2.404353141784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2021 , TFLOPS: 102.10676436833663, Tokens per sec: 83434.06562731722, Loss: 2.364633798599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2022 , TFLOPS: 102.79142849080007, Tokens per sec: 83993.52230660457, Loss: 2.378934383392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2023 , TFLOPS: 100.77943398902383, Tokens per sec: 82349.46980585706, Loss: 2.3815953731536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2024 , TFLOPS: 102.24373972423402, Tokens per sec: 83545.99171668034, Loss: 2.3671815395355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2025 , TFLOPS: 102.69576904468988, Tokens per sec: 83915.35651069472, Loss: 2.399782419204712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2026 , TFLOPS: 102.681202591071, Tokens per sec: 83903.45388646885, Loss: 2.4011213779449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2027 , TFLOPS: 100.38398186843123, Tokens per sec: 82026.33569827765, Loss: 2.3954944610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2028 , TFLOPS: 103.46819537310192, Tokens per sec: 84546.52594766283, Loss: 2.376378059387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2029 , TFLOPS: 102.03016440242025, Tokens per sec: 83371.4738233084, Loss: 2.385153293609619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2030 , TFLOPS: 99.69740410705518, Tokens per sec: 81465.31533537337, Loss: 2.378011465072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2031 , TFLOPS: 103.39706722622807, Tokens per sec: 84488.40530785082, Loss: 2.3902106285095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2032 , TFLOPS: 100.37286163359458, Tokens per sec: 82017.2490681323, Loss: 2.3845913410186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2033 , TFLOPS: 101.55354640618623, Tokens per sec: 82982.01698934687, Loss: 2.3942041397094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2034 , TFLOPS: 101.59533552237275, Tokens per sec: 83016.16395193042, Loss: 2.3753342628479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2035 , TFLOPS: 101.53270444790606, Tokens per sec: 82964.9864887164, Loss: 2.3774518966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2036 , TFLOPS: 100.91017244247895, Tokens per sec: 82456.29956168245, Loss: 2.3619229793548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2037 , TFLOPS: 101.03007499820096, Tokens per sec: 82554.27502652927, Loss: 2.3826658725738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2038 , TFLOPS: 100.82717170951788, Tokens per sec: 82388.47752615101, Loss: 2.377593517303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2039 , TFLOPS: 101.13865904446271, Tokens per sec: 82643.00184593175, Loss: 2.3907313346862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2040 , TFLOPS: 100.32415039101265, Tokens per sec: 81977.44585787966, Loss: 2.3859946727752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2041 , TFLOPS: 100.99084225063622, Tokens per sec: 82522.21693855335, Loss: 2.3872053623199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2042 , TFLOPS: 101.81885543452577, Tokens per sec: 83198.80782606508, Loss: 2.38665509223938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2043 , TFLOPS: 100.567162679794, Tokens per sec: 82176.01745473575, Loss: 2.3673858642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2044 , TFLOPS: 102.14032485160998, Tokens per sec: 83461.48876212357, Loss: 2.3807897567749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2045 , TFLOPS: 102.62442488902167, Tokens per sec: 83857.05936453627, Loss: 2.3854727745056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2046 , TFLOPS: 101.57350464515143, Tokens per sec: 82998.3253801761, Loss: 2.406546115875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2047 , TFLOPS: 102.72560884955267, Tokens per sec: 83939.73938339303, Loss: 2.3948750495910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2048 , TFLOPS: 102.69438681395536, Tokens per sec: 83914.22705437976, Loss: 2.382852554321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2049 , TFLOPS: 101.97489191950062, Tokens per sec: 83326.30925467449, Loss: 2.3970110416412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2050 , TFLOPS: 102.81991431444087, Tokens per sec: 84016.7988063918, Loss: 2.3969178199768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2051 , TFLOPS: 101.55559597149502, Tokens per sec: 82983.6917419212, Loss: 2.392226457595825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2052 , TFLOPS: 103.3583570711826, Tokens per sec: 84456.77424367511, Loss: 2.384935140609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2053 , TFLOPS: 102.51923529758069, Tokens per sec: 83771.10624154887, Loss: 2.372373342514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2054 , TFLOPS: 102.7165189637411, Tokens per sec: 83932.3118037019, Loss: 2.3743503093719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2055 , TFLOPS: 100.26488768774131, Tokens per sec: 81929.02077747896, Loss: 2.3782575130462646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2056 , TFLOPS: 102.53230626222873, Tokens per sec: 83781.78686323935, Loss: 2.3759219646453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2057 , TFLOPS: 103.37379457919636, Tokens per sec: 84469.38862886997, Loss: 2.362746477127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2058 , TFLOPS: 103.3566064211163, Tokens per sec: 84455.3437424402, Loss: 2.395819664001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2059 , TFLOPS: 100.74119702662499, Tokens per sec: 82318.22539957399, Loss: 2.3769631385803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2060 , TFLOPS: 102.76191816535528, Tokens per sec: 83969.40866002027, Loss: 2.4038963317871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2061 , TFLOPS: 101.42924489281475, Tokens per sec: 82880.44702296541, Loss: 2.3699655532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2062 , TFLOPS: 100.86044520093947, Tokens per sec: 82415.66615252745, Loss: 2.3813929557800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2063 , TFLOPS: 103.30924347398714, Tokens per sec: 84416.64225911023, Loss: 2.3608145713806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2064 , TFLOPS: 101.59084143039145, Tokens per sec: 83012.49171368429, Loss: 2.3692283630371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2065 , TFLOPS: 101.41034632639594, Tokens per sec: 82865.00451785223, Loss: 2.3852224349975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2066 , TFLOPS: 102.10950781364075, Tokens per sec: 83436.30736709779, Loss: 2.4065022468566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2067 , TFLOPS: 101.96011082743826, Tokens per sec: 83314.23124385117, Loss: 2.39367938041687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2068 , TFLOPS: 99.80265787902107, Tokens per sec: 81551.32090191981, Loss: 2.4027419090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2069 , TFLOPS: 103.43470421738672, Tokens per sec: 84519.1594621889, Loss: 2.388947010040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2070 , TFLOPS: 100.43164826105622, Tokens per sec: 82065.28513473363, Loss: 2.3638079166412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2071 , TFLOPS: 101.46129305022627, Tokens per sec: 82906.63439737934, Loss: 2.392193555831909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2072 , TFLOPS: 101.09621003637648, Tokens per sec: 82608.31566868981, Loss: 2.365006923675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2073 , TFLOPS: 101.76295971604273, Tokens per sec: 83153.13399560879, Loss: 2.384904384613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2074 , TFLOPS: 100.71161950800202, Tokens per sec: 82294.05684771409, Loss: 2.3742783069610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2075 , TFLOPS: 100.34201043766703, Tokens per sec: 81992.03976175946, Loss: 2.373349189758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2076 , TFLOPS: 101.57245609535899, Tokens per sec: 82997.46858314864, Loss: 2.4076969623565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2077 , TFLOPS: 101.85122692986764, Tokens per sec: 83225.25940822533, Loss: 2.375764846801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2078 , TFLOPS: 101.38437650669934, Tokens per sec: 82843.7839096555, Loss: 2.3782670497894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2079 , TFLOPS: 101.5865249553562, Tokens per sec: 83008.96461081703, Loss: 2.3813867568969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2080 , TFLOPS: 102.15724703053093, Tokens per sec: 83475.31631013553, Loss: 2.397003650665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2081 , TFLOPS: 101.6509361038141, Tokens per sec: 83061.59661831245, Loss: 2.363166093826294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2082 , TFLOPS: 101.994145209208, Tokens per sec: 83342.04161331816, Loss: 2.365671396255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2083 , TFLOPS: 100.88391121707228, Tokens per sec: 82434.84083837856, Loss: 2.396169662475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2084 , TFLOPS: 101.5422280377922, Tokens per sec: 82972.76845917138, Loss: 2.373464822769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2085 , TFLOPS: 102.63072764740346, Tokens per sec: 83862.20951065747, Loss: 2.3787176609039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2086 , TFLOPS: 102.22756028692682, Tokens per sec: 83532.77108196085, Loss: 2.370980739593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2087 , TFLOPS: 101.44738717539826, Tokens per sec: 82895.2715490883, Loss: 2.400669574737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2088 , TFLOPS: 101.98810780731942, Tokens per sec: 83337.10829681845, Loss: 2.3810901641845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2089 , TFLOPS: 102.27335850744272, Tokens per sec: 83570.19398689547, Loss: 2.3971755504608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2090 , TFLOPS: 103.18949489704379, Tokens per sec: 84318.79261428722, Loss: 2.3940112590789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2091 , TFLOPS: 102.2211488272887, Tokens per sec: 83527.53211324489, Loss: 2.3866875171661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2092 , TFLOPS: 102.64490558876905, Tokens per sec: 83873.79467151998, Loss: 2.358980655670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2093 , TFLOPS: 100.0601997587832, Tokens per sec: 81761.76500159124, Loss: 2.4090054035186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2094 , TFLOPS: 103.35373182656949, Tokens per sec: 84452.99483723735, Loss: 2.373680353164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2095 , TFLOPS: 103.46507732244257, Tokens per sec: 84543.9781082033, Loss: 2.3941597938537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2096 , TFLOPS: 101.86658566948147, Tokens per sec: 83237.80942973294, Loss: 2.3671815395355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2097 , TFLOPS: 101.51501330683512, Tokens per sec: 82950.53060193684, Loss: 2.393402099609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2098 , TFLOPS: 102.04638457819397, Tokens per sec: 83384.7277464776, Loss: 2.3925466537475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2099 , TFLOPS: 101.33432837867768, Tokens per sec: 82802.88829589558, Loss: 2.366069793701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2100 , TFLOPS: 101.58502711240347, Tokens per sec: 83007.74068478245, Loss: 2.3719615936279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2101 , TFLOPS: 102.58030332567778, Tokens per sec: 83821.0064993379, Loss: 2.3868484497070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2102 , TFLOPS: 101.63314730891574, Tokens per sec: 83047.06093607668, Loss: 2.3859856128692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2103 , TFLOPS: 101.33320745516762, Tokens per sec: 82801.97236044037, Loss: 2.3829379081726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2104 , TFLOPS: 102.13443722301557, Tokens per sec: 83456.67782923805, Loss: 2.3772544860839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2105 , TFLOPS: 101.52413811001206, Tokens per sec: 82957.98671350606, Loss: 2.3709747791290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2106 , TFLOPS: 100.15661216935847, Tokens per sec: 81840.54606414853, Loss: 2.3779773712158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2107 , TFLOPS: 102.76578733424158, Tokens per sec: 83972.57025751853, Loss: 2.3775551319122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2108 , TFLOPS: 100.91353568180223, Tokens per sec: 82459.04775111096, Loss: 2.370739221572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2109 , TFLOPS: 101.41084805219712, Tokens per sec: 82865.41449091994, Loss: 2.3655314445495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2110 , TFLOPS: 101.65431638918577, Tokens per sec: 83064.35873650592, Loss: 2.3942646980285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2111 , TFLOPS: 101.46512644337395, Tokens per sec: 82909.76676160062, Loss: 2.3703105449676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2112 , TFLOPS: 101.49463678057494, Tokens per sec: 82933.88041779121, Loss: 2.4037721157073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2113 , TFLOPS: 100.95353931265669, Tokens per sec: 82491.73574766722, Loss: 2.399029016494751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2114 , TFLOPS: 102.22510602427352, Tokens per sec: 83530.7656407685, Loss: 2.3982901573181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2115 , TFLOPS: 100.35996684352335, Tokens per sec: 82006.71240322368, Loss: 2.3782150745391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2116 , TFLOPS: 101.22779698282584, Tokens per sec: 82715.83874998294, Loss: 2.3929498195648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2117 , TFLOPS: 101.46111467781304, Tokens per sec: 82906.48864468877, Loss: 2.3842854499816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2118 , TFLOPS: 102.14967718100533, Tokens per sec: 83469.1307912229, Loss: 2.3768324851989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2119 , TFLOPS: 101.70498663305456, Tokens per sec: 83105.76269713923, Loss: 2.387993812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2120 , TFLOPS: 102.17000769443624, Tokens per sec: 83485.74337710127, Loss: 2.3690474033355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2121 , TFLOPS: 101.88592676327796, Tokens per sec: 83253.6135353581, Loss: 2.3721323013305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2122 , TFLOPS: 101.63967673857682, Tokens per sec: 83052.39630113523, Loss: 2.3989005088806152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2123 , TFLOPS: 102.75791783942255, Tokens per sec: 83966.13989072308, Loss: 2.3635454177856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2124 , TFLOPS: 102.68355901056812, Tokens per sec: 83905.37937749959, Loss: 2.3904216289520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2125 , TFLOPS: 101.44891199870547, Tokens per sec: 82896.51752146508, Loss: 2.383255958557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2126 , TFLOPS: 102.67186700089955, Tokens per sec: 83895.8255354203, Loss: 2.373201847076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2127 , TFLOPS: 102.86034839484738, Tokens per sec: 84049.83853435778, Loss: 2.377199649810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2128 , TFLOPS: 103.45360057340186, Tokens per sec: 84534.60016112433, Loss: 2.404367685317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2129 , TFLOPS: 102.17752598394672, Tokens per sec: 83491.88676499827, Loss: 2.37475848197937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2130 , TFLOPS: 102.67326400194648, Tokens per sec: 83896.96706093775, Loss: 2.370692491531372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2131 , TFLOPS: 100.82914231720851, Tokens per sec: 82390.08776042296, Loss: 2.3922722339630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2132 , TFLOPS: 102.64267903229397, Tokens per sec: 83871.97529490752, Loss: 2.377377986907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2133 , TFLOPS: 103.3651509861581, Tokens per sec: 84462.32572649253, Loss: 2.369347333908081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2134 , TFLOPS: 102.61876483305083, Tokens per sec: 83852.4343870997, Loss: 2.3882124423980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2135 , TFLOPS: 102.06836734502029, Tokens per sec: 83402.69043113814, Loss: 2.370698928833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2136 , TFLOPS: 102.02251872205504, Tokens per sec: 83365.22634106467, Loss: 2.392242431640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2137 , TFLOPS: 102.13676735977452, Tokens per sec: 83458.58184396684, Loss: 2.3710851669311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2138 , TFLOPS: 100.92121741879404, Tokens per sec: 82465.32469615249, Loss: 2.4023756980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2139 , TFLOPS: 103.3473852628951, Tokens per sec: 84447.80889668393, Loss: 2.386460304260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2140 , TFLOPS: 102.09924407504214, Tokens per sec: 83427.92060207673, Loss: 2.3774924278259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2141 , TFLOPS: 101.36787614937967, Tokens per sec: 82830.30104293261, Loss: 2.4129889011383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2142 , TFLOPS: 102.65972290510851, Tokens per sec: 83885.9022821326, Loss: 2.3806378841400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2143 , TFLOPS: 101.37013226574855, Tokens per sec: 82832.14457369522, Loss: 2.3849308490753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2144 , TFLOPS: 100.99792868309378, Tokens per sec: 82528.00744493541, Loss: 2.36002254486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2145 , TFLOPS: 102.73229542746309, Tokens per sec: 83945.20315833145, Loss: 2.3883509635925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2146 , TFLOPS: 101.60434501053086, Tokens per sec: 83023.52583662927, Loss: 2.3919668197631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2147 , TFLOPS: 101.5487666888786, Tokens per sec: 82978.11135929394, Loss: 2.37734055519104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2148 , TFLOPS: 102.07217755821577, Tokens per sec: 83405.80385442376, Loss: 2.3586277961730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2149 , TFLOPS: 102.741543804037, Tokens per sec: 83952.76024490438, Loss: 2.339491844177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2150 , TFLOPS: 100.6543448509199, Tokens per sec: 82247.25625103146, Loss: 2.3868014812469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2151 , TFLOPS: 101.52372456828931, Tokens per sec: 82957.64879791888, Loss: 2.3956029415130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2152 , TFLOPS: 101.65483583670637, Tokens per sec: 83064.78319044677, Loss: 2.357626438140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2153 , TFLOPS: 100.44038656440216, Tokens per sec: 82072.42542734153, Loss: 2.364478588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2154 , TFLOPS: 101.45679310866494, Tokens per sec: 82902.95737929076, Loss: 2.4002976417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2155 , TFLOPS: 101.60553527945821, Tokens per sec: 83024.49843601006, Loss: 2.3601126670837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2156 , TFLOPS: 102.04283961931147, Tokens per sec: 83381.83106931933, Loss: 2.3843441009521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2157 , TFLOPS: 101.12763450702045, Tokens per sec: 82633.99341259083, Loss: 2.3598477840423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2158 , TFLOPS: 102.20292098849895, Tokens per sec: 83512.63767694346, Loss: 2.387395143508911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2159 , TFLOPS: 101.53443658603683, Tokens per sec: 82966.40186337226, Loss: 2.359954833984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2160 , TFLOPS: 101.56849932356954, Tokens per sec: 82994.23540306311, Loss: 2.375831365585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2161 , TFLOPS: 101.98900281049482, Tokens per sec: 83337.83962695251, Loss: 2.346951484680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2162 , TFLOPS: 102.12191916518566, Tokens per sec: 83446.44900194174, Loss: 2.3600411415100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2163 , TFLOPS: 100.90400145691174, Tokens per sec: 82451.25709052016, Loss: 2.3779711723327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2164 , TFLOPS: 102.26747395347147, Tokens per sec: 83565.38556636366, Loss: 2.3727362155914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2165 , TFLOPS: 102.8194978395762, Tokens per sec: 84016.45849405885, Loss: 2.3674983978271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2166 , TFLOPS: 102.6626796137696, Tokens per sec: 83888.31828489203, Loss: 2.376955032348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2167 , TFLOPS: 102.12357139062942, Tokens per sec: 83447.79907788392, Loss: 2.3762197494506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2168 , TFLOPS: 101.95776322871541, Tokens per sec: 83312.31296050217, Loss: 2.375429630279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2169 , TFLOPS: 100.89682008340914, Tokens per sec: 82445.3890053662, Loss: 2.3842995166778564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2170 , TFLOPS: 102.68011010679167, Tokens per sec: 83902.56118944117, Loss: 2.3799641132354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2171 , TFLOPS: 103.40617632761337, Tokens per sec: 84495.84858908186, Loss: 2.3968899250030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2172 , TFLOPS: 101.92728988637464, Tokens per sec: 83287.4124080219, Loss: 2.392765522003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2173 , TFLOPS: 101.37586135065835, Tokens per sec: 82836.8259564548, Loss: 2.390679359436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2174 , TFLOPS: 101.4041173147092, Tokens per sec: 82859.91463205396, Loss: 2.365320920944214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2175 , TFLOPS: 101.47145363512952, Tokens per sec: 82914.9368728605, Loss: 2.3661856651306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2176 , TFLOPS: 101.57836556735613, Tokens per sec: 83002.29736484084, Loss: 2.3815526962280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2177 , TFLOPS: 102.66938141243097, Tokens per sec: 83893.79449709787, Loss: 2.37233829498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2178 , TFLOPS: 102.12502589764082, Tokens per sec: 83448.98759300527, Loss: 2.3807690143585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2179 , TFLOPS: 100.59928085341251, Tokens per sec: 82202.26204119477, Loss: 2.365391969680786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2180 , TFLOPS: 102.60019352090521, Tokens per sec: 83837.2592898774, Loss: 2.3551690578460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2181 , TFLOPS: 102.02313524409446, Tokens per sec: 83365.73011709379, Loss: 2.369297981262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2182 , TFLOPS: 100.79370271198013, Tokens per sec: 82361.12914669426, Loss: 2.3964614868164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2183 , TFLOPS: 102.82680978222726, Tokens per sec: 84022.43327062519, Loss: 2.375035285949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2184 , TFLOPS: 101.63912818442283, Tokens per sec: 83051.94806341495, Loss: 2.391602039337158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2185 , TFLOPS: 100.86286490685485, Tokens per sec: 82417.64335651968, Loss: 2.3612051010131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2186 , TFLOPS: 101.9166387055001, Tokens per sec: 83278.70905394353, Loss: 2.3820223808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2187 , TFLOPS: 102.86117850446503, Tokens per sec: 84050.51683829528, Loss: 2.3595693111419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2188 , TFLOPS: 100.80902690918144, Tokens per sec: 82373.65094270738, Loss: 2.364776611328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2189 , TFLOPS: 101.58800674150976, Tokens per sec: 83010.17541642766, Loss: 2.387488842010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2190 , TFLOPS: 102.22279668342124, Tokens per sec: 83528.87861891038, Loss: 2.381350040435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2191 , TFLOPS: 99.17560295568039, Tokens per sec: 81038.9381822284, Loss: 2.3780782222747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2192 , TFLOPS: 101.48936684090853, Tokens per sec: 82929.57421442946, Loss: 2.381903648376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2193 , TFLOPS: 102.2454614395226, Tokens per sec: 83547.39857456353, Loss: 2.368762254714966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2194 , TFLOPS: 101.50678879982603, Tokens per sec: 82943.81014554207, Loss: 2.3773388862609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2195 , TFLOPS: 101.15263930254716, Tokens per sec: 82654.42547469643, Loss: 2.3869659900665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2196 , TFLOPS: 101.51760235406745, Tokens per sec: 82952.64617907841, Loss: 2.375110149383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2197 , TFLOPS: 102.85637194988249, Tokens per sec: 84046.58927881418, Loss: 2.387662172317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2198 , TFLOPS: 101.00999186583772, Tokens per sec: 82537.86458208943, Loss: 2.3700783252716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2199 , TFLOPS: 102.1033667691616, Tokens per sec: 83431.28936156932, Loss: 2.372453212738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2200 , TFLOPS: 101.96405175029301, Tokens per sec: 83317.45147336418, Loss: 2.3636903762817383 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/optimizer.pt +[rank0]:[2024-08-27 07:59:14,763] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007717444997979328, 'preprocessing_with_comm': 0.001664538009208627, 'state_converting': 3.356862018001266, : 3.367862041995977}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200, took 15.63s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000200 + eval ppl=8.721463203430176, eval loss=2.1657869815826416 +------------------------------------------------------------------ +iteration: 2201 , TFLOPS: 101.80947290677597, Tokens per sec: 83191.14112111296, Loss: 2.384917974472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2202 , TFLOPS: 101.51516006644857, Tokens per sec: 82950.65052299494, Loss: 2.3869524002075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2203 , TFLOPS: 102.64091864564581, Tokens per sec: 83870.53683766093, Loss: 2.375627279281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2204 , TFLOPS: 101.5691251096087, Tokens per sec: 82994.74674894488, Loss: 2.398959159851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2205 , TFLOPS: 101.44964300579987, Tokens per sec: 82897.1148461797, Loss: 2.3832292556762695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2206 , TFLOPS: 101.74004819434066, Tokens per sec: 83134.41240143098, Loss: 2.3945112228393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2207 , TFLOPS: 102.28124115984448, Tokens per sec: 83576.63510509048, Loss: 2.384032964706421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2208 , TFLOPS: 101.64966874465867, Tokens per sec: 83060.56102652183, Loss: 2.3844685554504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2209 , TFLOPS: 101.88551842149239, Tokens per sec: 83253.27986877346, Loss: 2.3546481132507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2210 , TFLOPS: 102.67383544256577, Tokens per sec: 83897.43399977796, Loss: 2.3875060081481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2211 , TFLOPS: 103.28706903363113, Tokens per sec: 84398.52295307445, Loss: 2.3507468700408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2212 , TFLOPS: 102.5990571885777, Tokens per sec: 83836.33076348082, Loss: 2.376735210418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2213 , TFLOPS: 101.82489925710519, Tokens per sec: 83203.74638907655, Loss: 2.3720407485961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2214 , TFLOPS: 101.92696248146099, Tokens per sec: 83287.14487703881, Loss: 2.3583147525787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2215 , TFLOPS: 101.91738636014541, Tokens per sec: 83279.31998180057, Loss: 2.3724048137664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2216 , TFLOPS: 102.68119448134544, Tokens per sec: 83903.44725980335, Loss: 2.3799102306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2217 , TFLOPS: 101.32263562246621, Tokens per sec: 82793.33384379685, Loss: 2.377264976501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2218 , TFLOPS: 102.64295826272202, Tokens per sec: 83872.2034612783, Loss: 2.3824684619903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2219 , TFLOPS: 102.66680153070212, Tokens per sec: 83891.68640932518, Loss: 2.3849050998687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2220 , TFLOPS: 101.71827490660846, Tokens per sec: 83116.6208875307, Loss: 2.3461177349090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2221 , TFLOPS: 103.08391389434594, Tokens per sec: 84232.51965908601, Loss: 2.4088964462280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2222 , TFLOPS: 101.27774392224961, Tokens per sec: 82756.65167993472, Loss: 2.3687691688537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2223 , TFLOPS: 102.63659197575491, Tokens per sec: 83867.00140431468, Loss: 2.35502290725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2224 , TFLOPS: 101.90344511567369, Tokens per sec: 83267.92823206316, Loss: 2.369281530380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2225 , TFLOPS: 101.46442133503365, Tokens per sec: 82909.19059942465, Loss: 2.3901259899139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2226 , TFLOPS: 103.27708545529163, Tokens per sec: 84390.36511421308, Loss: 2.390474557876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2227 , TFLOPS: 100.34688228253624, Tokens per sec: 81996.02067161455, Loss: 2.3739495277404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2228 , TFLOPS: 102.63562174154687, Tokens per sec: 83866.20860096723, Loss: 2.373051404953003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2229 , TFLOPS: 103.27005928707209, Tokens per sec: 84384.62385129124, Loss: 2.3659849166870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2230 , TFLOPS: 99.84121307558887, Tokens per sec: 81582.82534553442, Loss: 2.366636276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2231 , TFLOPS: 103.29634021048557, Tokens per sec: 84406.09867034355, Loss: 2.3666508197784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2232 , TFLOPS: 102.88967518836348, Tokens per sec: 84073.80221227864, Loss: 2.363272190093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2233 , TFLOPS: 100.88931469933378, Tokens per sec: 82439.25616283245, Loss: 2.3612446784973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2234 , TFLOPS: 103.32169637726054, Tokens per sec: 84426.81784694118, Loss: 2.3691484928131104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2235 , TFLOPS: 102.04686227093741, Tokens per sec: 83385.11808151516, Loss: 2.3783504962921143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2236 , TFLOPS: 100.62640945461081, Tokens per sec: 82224.42951958622, Loss: 2.3712079524993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2237 , TFLOPS: 102.76423434891097, Tokens per sec: 83971.30127322751, Loss: 2.356055736541748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2238 , TFLOPS: 101.59795982191733, Tokens per sec: 83018.30833464378, Loss: 2.374048948287964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2239 , TFLOPS: 102.56521816524615, Tokens per sec: 83808.68002642263, Loss: 2.3535139560699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2240 , TFLOPS: 101.460960624284, Tokens per sec: 82906.3627635845, Loss: 2.3608126640319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2241 , TFLOPS: 102.60010482229158, Tokens per sec: 83837.18681195691, Loss: 2.387293815612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2242 , TFLOPS: 100.15627869153056, Tokens per sec: 81840.27357083086, Loss: 2.3725006580352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2243 , TFLOPS: 101.48271795730238, Tokens per sec: 82924.14124047739, Loss: 2.369022846221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2244 , TFLOPS: 102.73011994123937, Tokens per sec: 83943.42551253604, Loss: 2.3807241916656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2245 , TFLOPS: 102.20986170574253, Tokens per sec: 83518.30911567315, Loss: 2.372159719467163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2246 , TFLOPS: 101.59854242219765, Tokens per sec: 83018.7843923303, Loss: 2.389925479888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2247 , TFLOPS: 101.87517658211854, Tokens per sec: 83244.82928559846, Loss: 2.3819854259490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2248 , TFLOPS: 102.71317964490329, Tokens per sec: 83929.58316031781, Loss: 2.3789193630218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2249 , TFLOPS: 103.41037990354076, Tokens per sec: 84499.2834391818, Loss: 2.3763628005981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2250 , TFLOPS: 102.72096403359245, Tokens per sec: 83935.94398470373, Loss: 2.3694164752960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2251 , TFLOPS: 99.92524461791095, Tokens per sec: 81651.48968193014, Loss: 2.396425247192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2252 , TFLOPS: 101.97010342651473, Tokens per sec: 83322.39645378894, Loss: 2.385025978088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2253 , TFLOPS: 102.78580947484409, Tokens per sec: 83988.93086402059, Loss: 2.3869881629943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2254 , TFLOPS: 101.3472998364642, Tokens per sec: 82813.48760796798, Loss: 2.374596118927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2255 , TFLOPS: 101.89251084729023, Tokens per sec: 83258.99355989392, Loss: 2.3670287132263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2256 , TFLOPS: 103.35196040408992, Tokens per sec: 84451.54736232883, Loss: 2.3665738105773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2257 , TFLOPS: 101.66388965672226, Tokens per sec: 83072.18129984783, Loss: 2.3782293796539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2258 , TFLOPS: 102.62696493672908, Tokens per sec: 83859.13490290464, Loss: 2.3876445293426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2259 , TFLOPS: 102.06505716973815, Tokens per sec: 83399.9856017036, Loss: 2.373030662536621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2260 , TFLOPS: 102.76628413187271, Tokens per sec: 83972.97620365165, Loss: 2.370354175567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2261 , TFLOPS: 101.8570222203293, Tokens per sec: 83229.99488924563, Loss: 2.373631000518799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2262 , TFLOPS: 102.05852217232163, Tokens per sec: 83394.64568708876, Loss: 2.3881537914276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2263 , TFLOPS: 101.26910316696416, Tokens per sec: 82749.59109635813, Loss: 2.383517265319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2264 , TFLOPS: 103.27728734912945, Tokens per sec: 84390.53008686517, Loss: 2.359347105026245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2265 , TFLOPS: 100.16388249763052, Tokens per sec: 81846.48683653453, Loss: 2.358551502227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2266 , TFLOPS: 102.53228771972246, Tokens per sec: 83781.77171168015, Loss: 2.371474266052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2267 , TFLOPS: 102.02792559417306, Tokens per sec: 83369.64443545758, Loss: 2.3603081703186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2268 , TFLOPS: 100.82120075047548, Tokens per sec: 82383.5985018113, Loss: 2.380868673324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2269 , TFLOPS: 102.62568725217514, Tokens per sec: 83858.09087396509, Loss: 2.3872833251953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2270 , TFLOPS: 102.23173535416586, Tokens per sec: 83536.18263687767, Loss: 2.3623385429382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2271 , TFLOPS: 101.0063373528958, Tokens per sec: 82534.87838548906, Loss: 2.3731772899627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2272 , TFLOPS: 102.3988387368033, Tokens per sec: 83672.72711245446, Loss: 2.3701229095458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2273 , TFLOPS: 101.602817883697, Tokens per sec: 83022.27798198159, Loss: 2.392500877380371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2274 , TFLOPS: 102.62493673230166, Tokens per sec: 83857.47760485638, Loss: 2.372544527053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2275 , TFLOPS: 102.24618759578846, Tokens per sec: 83547.99193554129, Loss: 2.3809585571289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2276 , TFLOPS: 102.08071864608424, Tokens per sec: 83412.78299718825, Loss: 2.3962323665618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2277 , TFLOPS: 102.11274958768327, Tokens per sec: 83438.95630411862, Loss: 2.3760509490966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2278 , TFLOPS: 100.89358236576902, Tokens per sec: 82442.7433829362, Loss: 2.3821616172790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2279 , TFLOPS: 102.01190416625701, Tokens per sec: 83356.55291427913, Loss: 2.3883004188537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2280 , TFLOPS: 101.57957032954648, Tokens per sec: 83003.28180703992, Loss: 2.374418020248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2281 , TFLOPS: 102.24034230647753, Tokens per sec: 83543.21559917419, Loss: 2.370105266571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2282 , TFLOPS: 102.75158562805633, Tokens per sec: 83960.96567781013, Loss: 2.3722338676452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2283 , TFLOPS: 102.82257637595272, Tokens per sec: 84018.9740453807, Loss: 2.3728814125061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2284 , TFLOPS: 100.92991512474954, Tokens per sec: 82472.43181558758, Loss: 2.364598274230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2285 , TFLOPS: 102.745101077958, Tokens per sec: 83955.6669849976, Loss: 2.373757839202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2286 , TFLOPS: 101.99231472742996, Tokens per sec: 83340.54587953653, Loss: 2.3619391918182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2287 , TFLOPS: 102.70117271873781, Tokens per sec: 83919.77199186222, Loss: 2.350385904312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2288 , TFLOPS: 102.7618707198439, Tokens per sec: 83969.36989107156, Loss: 2.391317844390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2289 , TFLOPS: 102.06439976037008, Tokens per sec: 83399.44841558574, Loss: 2.366478204727173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2290 , TFLOPS: 101.42181840173224, Tokens per sec: 82874.378645927, Loss: 2.3516383171081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2291 , TFLOPS: 101.95691426168568, Tokens per sec: 83311.61924768784, Loss: 2.370267152786255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2292 , TFLOPS: 101.90154132928608, Tokens per sec: 83266.37259919802, Loss: 2.3494865894317627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2293 , TFLOPS: 101.30789182653459, Tokens per sec: 82781.28630860205, Loss: 2.3719444274902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2294 , TFLOPS: 102.74189605433047, Tokens per sec: 83953.04807768691, Loss: 2.3656017780303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2295 , TFLOPS: 101.941126327416, Tokens per sec: 83298.71852017847, Loss: 2.385716199874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2296 , TFLOPS: 101.6295968378205, Tokens per sec: 83044.15975474751, Loss: 2.3817944526672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2297 , TFLOPS: 102.0018307461142, Tokens per sec: 83348.32166336753, Loss: 2.3903274536132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2298 , TFLOPS: 101.96250183228528, Tokens per sec: 83316.18499546162, Loss: 2.375795602798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2299 , TFLOPS: 101.67604373883906, Tokens per sec: 83082.11271321937, Loss: 2.3644559383392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2300 , TFLOPS: 102.15075795864239, Tokens per sec: 83470.01392245182, Loss: 2.37858247756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2301 , TFLOPS: 101.37892928201154, Tokens per sec: 82839.33284213906, Loss: 2.3675570487976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2302 , TFLOPS: 102.15731355494034, Tokens per sec: 83475.37066894295, Loss: 2.379218816757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2303 , TFLOPS: 100.23377092936538, Tokens per sec: 81903.59447319336, Loss: 2.332897663116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2304 , TFLOPS: 102.12134892162459, Tokens per sec: 83445.98304124862, Loss: 2.374591112136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2305 , TFLOPS: 102.00767574069035, Tokens per sec: 83353.09775889925, Loss: 2.3724513053894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2306 , TFLOPS: 100.81796734593432, Tokens per sec: 82380.95640372547, Loss: 2.384341239929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2307 , TFLOPS: 102.0822387748009, Tokens per sec: 83414.02513349424, Loss: 2.370676279067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2308 , TFLOPS: 102.15066720187299, Tokens per sec: 83469.93976275927, Loss: 2.3567051887512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2309 , TFLOPS: 100.8109019282289, Tokens per sec: 82375.18306903835, Loss: 2.3832240104675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2310 , TFLOPS: 102.09886106184582, Tokens per sec: 83427.60763213439, Loss: 2.3633532524108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2311 , TFLOPS: 102.1203761935528, Tokens per sec: 83445.18820010092, Loss: 2.3607897758483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2312 , TFLOPS: 102.00749497545543, Tokens per sec: 83352.95005097249, Loss: 2.382781982421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2313 , TFLOPS: 102.58772159381328, Tokens per sec: 83827.06815719462, Loss: 2.3825201988220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2314 , TFLOPS: 102.18141633698279, Tokens per sec: 83495.06567261064, Loss: 2.3831968307495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2315 , TFLOPS: 102.72173864421781, Tokens per sec: 83936.57693898595, Loss: 2.3436715602874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2316 , TFLOPS: 101.71923902282768, Tokens per sec: 83117.408691712, Loss: 2.366898775100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2317 , TFLOPS: 102.7740046765888, Tokens per sec: 83979.28485948367, Loss: 2.3830435276031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2318 , TFLOPS: 100.46296104181823, Tokens per sec: 82090.87161395694, Loss: 2.380028247833252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2319 , TFLOPS: 100.28082608327584, Tokens per sec: 81942.04445076102, Loss: 2.3459014892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2320 , TFLOPS: 102.67109862415249, Tokens per sec: 83895.1976750005, Loss: 2.348346710205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2321 , TFLOPS: 102.90404552544574, Tokens per sec: 84085.54458463393, Loss: 2.3727943897247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2322 , TFLOPS: 101.60160476482025, Tokens per sec: 83021.28671131894, Loss: 2.3705761432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2323 , TFLOPS: 102.19742696020539, Tokens per sec: 83508.14836499593, Loss: 2.3744664192199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2324 , TFLOPS: 102.75227186364313, Tokens per sec: 83961.52641857337, Loss: 2.3484320640563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2325 , TFLOPS: 102.0223173349928, Tokens per sec: 83365.06178251196, Loss: 2.3492929935455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2326 , TFLOPS: 102.15342991786328, Tokens per sec: 83472.19724911357, Loss: 2.374084234237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2327 , TFLOPS: 102.21336001015999, Tokens per sec: 83521.16767026704, Loss: 2.371673345565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2328 , TFLOPS: 102.11701618824159, Tokens per sec: 83442.4426532669, Loss: 2.3740739822387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2329 , TFLOPS: 101.6510390323099, Tokens per sec: 83061.68072383598, Loss: 2.399468421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2330 , TFLOPS: 102.02289853716607, Tokens per sec: 83365.53669776922, Loss: 2.368394374847412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2331 , TFLOPS: 100.80853471568771, Tokens per sec: 82373.24875873346, Loss: 2.377511501312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2332 , TFLOPS: 102.74419501719828, Tokens per sec: 83954.92661942677, Loss: 2.3683934211730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2333 , TFLOPS: 102.68811322225497, Tokens per sec: 83909.10074110476, Loss: 2.3708038330078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2334 , TFLOPS: 100.92553061233498, Tokens per sec: 82468.84911762626, Loss: 2.3707427978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2335 , TFLOPS: 102.15559734836437, Tokens per sec: 83473.96831237063, Loss: 2.3944098949432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2336 , TFLOPS: 102.11447566802845, Tokens per sec: 83440.36672880192, Loss: 2.3638527393341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2337 , TFLOPS: 101.98639999550473, Tokens per sec: 83335.71279982169, Loss: 2.368445634841919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2338 , TFLOPS: 101.47160237864296, Tokens per sec: 82915.05841501433, Loss: 2.3620481491088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2339 , TFLOPS: 102.07837382940234, Tokens per sec: 83410.86698711653, Loss: 2.3488452434539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2340 , TFLOPS: 102.44673670868201, Tokens per sec: 83711.86577828004, Loss: 2.358849287033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2341 , TFLOPS: 100.99801598631913, Tokens per sec: 82528.07878264824, Loss: 2.3736231327056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2342 , TFLOPS: 101.04726969410221, Tokens per sec: 82568.32525518103, Loss: 2.358293056488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2343 , TFLOPS: 103.27659666707935, Tokens per sec: 84389.9657127823, Loss: 2.3887035846710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2344 , TFLOPS: 99.99650870746964, Tokens per sec: 81709.72140400941, Loss: 2.3867321014404297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2345 , TFLOPS: 103.2693180632304, Tokens per sec: 84384.0181782094, Loss: 2.357272148132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2346 , TFLOPS: 100.952744429831, Tokens per sec: 82491.08622844824, Loss: 2.3620657920837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2347 , TFLOPS: 102.04111372929906, Tokens per sec: 83380.42080016184, Loss: 2.3708226680755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2348 , TFLOPS: 101.44062633609309, Tokens per sec: 82889.74709325256, Loss: 2.3907034397125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2349 , TFLOPS: 102.1466066908862, Tokens per sec: 83466.62181470511, Loss: 2.3357176780700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2350 , TFLOPS: 101.54234970935913, Tokens per sec: 82972.86788014082, Loss: 2.3559248447418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2351 , TFLOPS: 101.45893886753359, Tokens per sec: 82904.71073409908, Loss: 2.3629660606384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2352 , TFLOPS: 101.95574729434415, Tokens per sec: 83310.66568863264, Loss: 2.366262197494507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2353 , TFLOPS: 101.65467235792096, Tokens per sec: 83064.64960772291, Loss: 2.3611252307891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2354 , TFLOPS: 101.99189544505506, Tokens per sec: 83340.20327311472, Loss: 2.3630614280700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2355 , TFLOPS: 101.58544315910282, Tokens per sec: 83008.08064725048, Loss: 2.355844020843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2356 , TFLOPS: 102.129832979916, Tokens per sec: 83452.91558367763, Loss: 2.366231918334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2357 , TFLOPS: 101.53346725712953, Tokens per sec: 82965.60979976936, Loss: 2.354790210723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2358 , TFLOPS: 102.87313794836192, Tokens per sec: 84060.28920776691, Loss: 2.389164447784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2359 , TFLOPS: 102.14733421858364, Tokens per sec: 83467.2162963149, Loss: 2.3967700004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2360 , TFLOPS: 102.93010051021784, Tokens per sec: 84106.83478340619, Loss: 2.370797872543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2361 , TFLOPS: 100.88552750513797, Tokens per sec: 82436.16154896401, Loss: 2.3722405433654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2362 , TFLOPS: 102.71745404778444, Tokens per sec: 83933.07588494492, Loss: 2.3744475841522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2363 , TFLOPS: 100.86517607348088, Tokens per sec: 82419.53187026453, Loss: 2.365372896194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2364 , TFLOPS: 103.26974858489531, Tokens per sec: 84384.36996854442, Loss: 2.371112585067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2365 , TFLOPS: 102.7566620657133, Tokens per sec: 83965.11376569876, Loss: 2.3720815181732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2366 , TFLOPS: 101.10775020232812, Tokens per sec: 82617.7454353592, Loss: 2.3583126068115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2367 , TFLOPS: 102.51363234672657, Tokens per sec: 83766.52792617318, Loss: 2.3561675548553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2368 , TFLOPS: 102.17170618096671, Tokens per sec: 83487.13125416824, Loss: 2.3824853897094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2369 , TFLOPS: 101.41074274654139, Tokens per sec: 82865.32844295783, Loss: 2.377903461456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2370 , TFLOPS: 102.71521695040155, Tokens per sec: 83931.24789508537, Loss: 2.367358922958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2371 , TFLOPS: 101.92724278133518, Tokens per sec: 83287.37391728157, Loss: 2.3605797290802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2372 , TFLOPS: 101.96222139027533, Tokens per sec: 83315.95583907608, Loss: 2.3735082149505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2373 , TFLOPS: 102.24005658415817, Tokens per sec: 83542.9821281119, Loss: 2.3535916805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2374 , TFLOPS: 102.3208866198008, Tokens per sec: 83609.03043098572, Loss: 2.381096839904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2375 , TFLOPS: 102.64558805100842, Tokens per sec: 83874.3523289839, Loss: 2.3569178581237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2376 , TFLOPS: 100.82275175130998, Tokens per sec: 82384.86586451948, Loss: 2.3542380332946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2377 , TFLOPS: 102.8255938098253, Tokens per sec: 84021.43966827364, Loss: 2.388862133026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2378 , TFLOPS: 103.35542100483588, Tokens per sec: 84454.37510828214, Loss: 2.358055353164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2379 , TFLOPS: 100.8487627417174, Tokens per sec: 82406.12011436385, Loss: 2.371023416519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2380 , TFLOPS: 102.04128564501508, Tokens per sec: 83380.56127691889, Loss: 2.364886522293091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2381 , TFLOPS: 102.59938690625752, Tokens per sec: 83836.60018428467, Loss: 2.3820297718048096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2382 , TFLOPS: 100.11510360164515, Tokens per sec: 81806.62834494439, Loss: 2.3500561714172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2383 , TFLOPS: 103.38894672311787, Tokens per sec: 84481.76983571911, Loss: 2.363518714904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2384 , TFLOPS: 100.22941389828821, Tokens per sec: 81900.03423094026, Loss: 2.366333484649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2385 , TFLOPS: 102.10613231652104, Tokens per sec: 83433.54916150824, Loss: 2.3563411235809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2386 , TFLOPS: 101.33207234003906, Tokens per sec: 82801.04482864836, Loss: 2.365035057067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2387 , TFLOPS: 101.48042353235572, Tokens per sec: 82922.26640678973, Loss: 2.366093635559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2388 , TFLOPS: 102.01215917051451, Tokens per sec: 83356.76128482232, Loss: 2.3805034160614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2389 , TFLOPS: 101.43196630783589, Tokens per sec: 82882.67076123465, Loss: 2.36738657951355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2390 , TFLOPS: 101.38507539441885, Tokens per sec: 82844.35498880208, Loss: 2.349705219268799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2391 , TFLOPS: 100.96767533762677, Tokens per sec: 82503.28665756393, Loss: 2.3900399208068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2392 , TFLOPS: 102.27419232927052, Tokens per sec: 83570.87532417546, Loss: 2.3733057975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2393 , TFLOPS: 101.09488405456578, Tokens per sec: 82607.23217481849, Loss: 2.364064931869507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2394 , TFLOPS: 102.19787098741878, Tokens per sec: 83508.51119106228, Loss: 2.363755702972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2395 , TFLOPS: 100.86925201056843, Tokens per sec: 82422.86242338271, Loss: 2.3633804321289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2396 , TFLOPS: 103.4427963626547, Tokens per sec: 84525.77176239771, Loss: 2.3797810077667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2397 , TFLOPS: 102.17357386093344, Tokens per sec: 83488.65738354743, Loss: 2.3753838539123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2398 , TFLOPS: 102.33854431229867, Tokens per sec: 83623.45898607517, Loss: 2.3585827350616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2399 , TFLOPS: 102.12629108769332, Tokens per sec: 83450.02141236606, Loss: 2.368044376373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2400 , TFLOPS: 102.1417476655391, Tokens per sec: 83462.65138001155, Loss: 2.3659887313842773 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/optimizer.pt +[rank0]:[2024-08-27 08:52:03,931] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007682872994337231, 'preprocessing_with_comm': 0.0015320419915951788, 'state_converting': 3.2840731159958523, : 3.294944765002583}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400, took 15.57s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000400 + eval ppl=8.024024963378906, eval loss=2.082440137863159 +------------------------------------------------------------------ +iteration: 2401 , TFLOPS: 100.16078478211554, Tokens per sec: 81843.95561344494, Loss: 2.376237630844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2402 , TFLOPS: 100.92529290022102, Tokens per sec: 82468.6548769387, Loss: 2.3772406578063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2403 , TFLOPS: 101.48810975531431, Tokens per sec: 82928.54701743017, Loss: 2.3863112926483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2404 , TFLOPS: 101.54376572513313, Tokens per sec: 82974.02494308125, Loss: 2.3811450004577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2405 , TFLOPS: 100.35471276892848, Tokens per sec: 82002.41916362029, Loss: 2.396630048751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2406 , TFLOPS: 102.24164221758011, Tokens per sec: 83544.2777899988, Loss: 2.37129545211792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2407 , TFLOPS: 101.2852048312034, Tokens per sec: 82762.74818070173, Loss: 2.3828113079071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2408 , TFLOPS: 102.19396578708216, Tokens per sec: 83505.32015133841, Loss: 2.378350019454956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2409 , TFLOPS: 101.45353490523122, Tokens per sec: 82900.29501739147, Loss: 2.3519439697265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2410 , TFLOPS: 102.08093476248624, Tokens per sec: 83412.95959146367, Loss: 2.340942621231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2411 , TFLOPS: 102.90162450938959, Tokens per sec: 84083.56631009193, Loss: 2.364417314529419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2412 , TFLOPS: 102.47788626148834, Tokens per sec: 83737.31887973893, Loss: 2.378101348876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2413 , TFLOPS: 101.3080802202977, Tokens per sec: 82781.44024999559, Loss: 2.335508108139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2414 , TFLOPS: 102.17126331743022, Tokens per sec: 83486.76937897225, Loss: 2.353280782699585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2415 , TFLOPS: 102.77480561868302, Tokens per sec: 83979.93932988693, Loss: 2.3764309883117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2416 , TFLOPS: 101.97434939095126, Tokens per sec: 83325.86594063089, Loss: 2.361870050430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2417 , TFLOPS: 100.8988243690745, Tokens per sec: 82447.02675877826, Loss: 2.3604674339294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2418 , TFLOPS: 102.24107967915369, Tokens per sec: 83543.81812536954, Loss: 2.381228446960449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2419 , TFLOPS: 102.62729733292578, Tokens per sec: 83859.40651239362, Loss: 2.3731470108032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2420 , TFLOPS: 103.39191854272616, Tokens per sec: 84484.19818602249, Loss: 2.370493173599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2421 , TFLOPS: 102.15932528092313, Tokens per sec: 83477.01450202992, Loss: 2.3745009899139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2422 , TFLOPS: 102.64345970679112, Tokens per sec: 83872.61320413549, Loss: 2.358396530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2423 , TFLOPS: 102.48530513544836, Tokens per sec: 83743.38103263047, Loss: 2.3618617057800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2424 , TFLOPS: 101.31637887506368, Tokens per sec: 82788.22129443125, Loss: 2.3632142543792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2425 , TFLOPS: 103.24620581955215, Tokens per sec: 84365.13256893781, Loss: 2.3561923503875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2426 , TFLOPS: 100.1239734565055, Tokens per sec: 81813.87613168116, Loss: 2.3560426235198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2427 , TFLOPS: 102.71932437185009, Tokens per sec: 83934.60417488539, Loss: 2.3539814949035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2428 , TFLOPS: 102.123183667663, Tokens per sec: 83447.48225946717, Loss: 2.379051685333252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2429 , TFLOPS: 102.84051070896254, Tokens per sec: 84033.62865055382, Loss: 2.3379249572753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2430 , TFLOPS: 101.54497792551417, Tokens per sec: 82975.01546321741, Loss: 2.347543478012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2431 , TFLOPS: 100.60265456489485, Tokens per sec: 82205.01878769406, Loss: 2.3794026374816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2432 , TFLOPS: 102.23416532922718, Tokens per sec: 83538.1682320534, Loss: 2.3610477447509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2433 , TFLOPS: 102.13847623506202, Tokens per sec: 83459.97820995461, Loss: 2.372196674346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2434 , TFLOPS: 100.21076532185698, Tokens per sec: 81884.79599908124, Loss: 2.3488597869873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2435 , TFLOPS: 102.24195448553118, Tokens per sec: 83544.53295218012, Loss: 2.359663724899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2436 , TFLOPS: 101.10192640060359, Tokens per sec: 82612.9866570521, Loss: 2.387772560119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2437 , TFLOPS: 101.07655604739178, Tokens per sec: 82592.25588835172, Loss: 2.351670265197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2438 , TFLOPS: 102.03723763141534, Tokens per sec: 83377.25354080142, Loss: 2.3659298419952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2439 , TFLOPS: 101.96064592048378, Tokens per sec: 83314.6684821532, Loss: 2.3653955459594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2440 , TFLOPS: 101.53911774898559, Tokens per sec: 82970.22696212154, Loss: 2.3747544288635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2441 , TFLOPS: 102.07485277983031, Tokens per sec: 83407.98984687135, Loss: 2.3614819049835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2442 , TFLOPS: 101.50038680021272, Tokens per sec: 82938.57890685592, Loss: 2.366032123565674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2443 , TFLOPS: 102.1689901561297, Tokens per sec: 83484.91192035723, Loss: 2.342136859893799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2444 , TFLOPS: 101.11751291795095, Tokens per sec: 82625.72280160927, Loss: 2.3609747886657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2445 , TFLOPS: 101.13570433361639, Tokens per sec: 82640.5874756382, Loss: 2.358048915863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2446 , TFLOPS: 101.5347653843073, Tokens per sec: 82966.67053290308, Loss: 2.3690545558929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2447 , TFLOPS: 101.7248070878804, Tokens per sec: 83121.95850100131, Loss: 2.3612186908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2448 , TFLOPS: 102.90196962788112, Tokens per sec: 84083.8483152955, Loss: 2.37477445602417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2449 , TFLOPS: 102.67994180351539, Tokens per sec: 83902.4236645016, Loss: 2.3564467430114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2450 , TFLOPS: 102.25536831304386, Tokens per sec: 83555.49373594337, Loss: 2.354412794113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2451 , TFLOPS: 102.05467466181649, Tokens per sec: 83391.5017872113, Loss: 2.3623054027557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2452 , TFLOPS: 103.40908231153618, Tokens per sec: 84498.22314335188, Loss: 2.3557422161102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2453 , TFLOPS: 103.37413045977696, Tokens per sec: 84469.66308553872, Loss: 2.3501906394958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2454 , TFLOPS: 101.77253451237547, Tokens per sec: 83160.95780816945, Loss: 2.3718883991241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2455 , TFLOPS: 101.98950648216096, Tokens per sec: 83338.25119003656, Loss: 2.340475559234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2456 , TFLOPS: 102.68648721811046, Tokens per sec: 83907.7720912612, Loss: 2.36122465133667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2457 , TFLOPS: 103.4436124613012, Tokens per sec: 84526.43861761, Loss: 2.352140188217163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2458 , TFLOPS: 102.64605489947301, Tokens per sec: 83874.73380288208, Loss: 2.380664110183716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2459 , TFLOPS: 101.5266543995126, Tokens per sec: 82960.042838432, Loss: 2.3663055896759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2460 , TFLOPS: 102.67922508777497, Tokens per sec: 83901.8380176202, Loss: 2.365220785140991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2461 , TFLOPS: 101.90207455955704, Tokens per sec: 83266.80831537899, Loss: 2.362565755844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2462 , TFLOPS: 101.452950402536, Tokens per sec: 82899.81740519276, Loss: 2.379629611968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2463 , TFLOPS: 103.28952393542284, Tokens per sec: 84400.52891652325, Loss: 2.371652603149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2464 , TFLOPS: 100.77323169877094, Tokens per sec: 82344.40175483028, Loss: 2.383291244506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2465 , TFLOPS: 102.02735455873804, Tokens per sec: 83369.17782770387, Loss: 2.3590452671051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2466 , TFLOPS: 102.15056197244228, Tokens per sec: 83469.85377708258, Loss: 2.351837635040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2467 , TFLOPS: 102.78798676767869, Tokens per sec: 83990.70998604425, Loss: 2.3616626262664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2468 , TFLOPS: 101.36598019456896, Tokens per sec: 82828.75180945056, Loss: 2.36366868019104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2469 , TFLOPS: 100.88931088305085, Tokens per sec: 82439.25304444942, Loss: 2.372143030166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2470 , TFLOPS: 101.99132367258909, Tokens per sec: 83339.73606311403, Loss: 2.3610382080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2471 , TFLOPS: 102.07988819015692, Tokens per sec: 83412.10441027221, Loss: 2.3890016078948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2472 , TFLOPS: 100.15554888644783, Tokens per sec: 81839.67722831092, Loss: 2.3673276901245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2473 , TFLOPS: 101.66068504757406, Tokens per sec: 83069.56273121886, Loss: 2.343742847442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2474 , TFLOPS: 102.10331568345048, Tokens per sec: 83431.24761811971, Loss: 2.3469600677490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2475 , TFLOPS: 99.46902749953232, Tokens per sec: 81278.70293042954, Loss: 2.364398241043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2476 , TFLOPS: 102.82676830218114, Tokens per sec: 84022.39937621172, Loss: 2.372152090072632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2477 , TFLOPS: 102.06146767413006, Tokens per sec: 83397.0525324404, Loss: 2.3771958351135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2478 , TFLOPS: 100.80080835230196, Tokens per sec: 82366.93534831668, Loss: 2.352792501449585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2479 , TFLOPS: 102.19950970149236, Tokens per sec: 83509.85022651605, Loss: 2.3734140396118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2480 , TFLOPS: 100.99639772459918, Tokens per sec: 82526.7564593391, Loss: 2.3573901653289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2481 , TFLOPS: 102.74976942070134, Tokens per sec: 83959.48160802702, Loss: 2.3364710807800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2482 , TFLOPS: 100.4866515524334, Tokens per sec: 82110.2297400286, Loss: 2.3555521965026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2483 , TFLOPS: 102.1300431588502, Tokens per sec: 83453.08732629525, Loss: 2.3877289295196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2484 , TFLOPS: 100.09208733106935, Tokens per sec: 81787.82115776539, Loss: 2.3751089572906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2485 , TFLOPS: 101.68263375174115, Tokens per sec: 83087.4975824037, Loss: 2.352595329284668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2486 , TFLOPS: 103.45329059889951, Tokens per sec: 84534.34687297902, Loss: 2.366083860397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2487 , TFLOPS: 102.25923112085478, Tokens per sec: 83558.65013564318, Loss: 2.3609235286712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2488 , TFLOPS: 102.87566787248691, Tokens per sec: 84062.35647389549, Loss: 2.350969076156616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2489 , TFLOPS: 102.01176317394376, Tokens per sec: 83356.43770583051, Loss: 2.372696876525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2490 , TFLOPS: 102.66669392225687, Tokens per sec: 83891.59847969447, Loss: 2.3608779907226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2491 , TFLOPS: 103.42882005570078, Tokens per sec: 84514.35136220351, Loss: 2.365638256072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2492 , TFLOPS: 102.72540294213682, Tokens per sec: 83939.571131143, Loss: 2.3457016944885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2493 , TFLOPS: 101.92108433024465, Tokens per sec: 83282.34168836287, Loss: 2.359835624694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2494 , TFLOPS: 101.7080958687197, Tokens per sec: 83108.30333364007, Loss: 2.335721492767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2495 , TFLOPS: 103.28636343743607, Tokens per sec: 84397.94639225982, Loss: 2.3485188484191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2496 , TFLOPS: 102.51554281423553, Tokens per sec: 83768.08901835456, Loss: 2.3807127475738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2497 , TFLOPS: 101.63078320186932, Tokens per sec: 83045.12916335152, Loss: 2.3797037601470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2498 , TFLOPS: 102.03383615707578, Tokens per sec: 83374.47410855681, Loss: 2.357947826385498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2499 , TFLOPS: 102.67702121810017, Tokens per sec: 83900.03717897627, Loss: 2.3505327701568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2500 , TFLOPS: 102.04814848495661, Tokens per sec: 83386.16908010034, Loss: 2.3649871349334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2501 , TFLOPS: 102.69742616927776, Tokens per sec: 83916.71058985367, Loss: 2.3711471557617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2502 , TFLOPS: 98.2608308262369, Tokens per sec: 80291.4543269306, Loss: 2.3721823692321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2503 , TFLOPS: 102.14006042152198, Tokens per sec: 83461.27268949164, Loss: 2.3527719974517822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2504 , TFLOPS: 102.80174202426505, Tokens per sec: 84001.94976029279, Loss: 2.3504252433776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2505 , TFLOPS: 102.28757197504851, Tokens per sec: 83581.80817716337, Loss: 2.3565008640289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2506 , TFLOPS: 102.00267081246952, Tokens per sec: 83349.0081032117, Loss: 2.375161647796631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2507 , TFLOPS: 99.76194384872322, Tokens per sec: 81518.05241969106, Loss: 2.366079092025757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2508 , TFLOPS: 102.13011186311735, Tokens per sec: 83453.14346632054, Loss: 2.357844829559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2509 , TFLOPS: 102.13127682931332, Tokens per sec: 83454.09539018816, Loss: 2.3749232292175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2510 , TFLOPS: 101.12193942338996, Tokens per sec: 82629.33981315188, Loss: 2.3720223903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2511 , TFLOPS: 100.88378640864825, Tokens per sec: 82434.73885420243, Loss: 2.353964328765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2512 , TFLOPS: 101.76857201268518, Tokens per sec: 83157.71994766871, Loss: 2.377763509750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2513 , TFLOPS: 100.92264213439071, Tokens per sec: 82466.48886794197, Loss: 2.3670198917388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2514 , TFLOPS: 101.56533746107071, Tokens per sec: 82991.65176382173, Loss: 2.369584560394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2515 , TFLOPS: 102.87302292034322, Tokens per sec: 84060.19521541175, Loss: 2.362208843231201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2516 , TFLOPS: 102.0527647601428, Tokens per sec: 83389.94115738847, Loss: 2.3419625759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2517 , TFLOPS: 101.92322002444737, Tokens per sec: 83284.08681907358, Loss: 2.358506917953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2518 , TFLOPS: 102.01926160743604, Tokens per sec: 83362.5648688639, Loss: 2.3768911361694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2519 , TFLOPS: 102.21511190383114, Tokens per sec: 83522.59918768349, Loss: 2.342315912246704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2520 , TFLOPS: 101.63888514299089, Tokens per sec: 83051.74946800448, Loss: 2.3556997776031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2521 , TFLOPS: 100.69957299830324, Tokens per sec: 82284.21333453448, Loss: 2.3622875213623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2522 , TFLOPS: 101.42328469417025, Tokens per sec: 82875.57679122382, Loss: 2.361558675765991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2523 , TFLOPS: 101.54356127233771, Tokens per sec: 82973.85787943903, Loss: 2.341590404510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2524 , TFLOPS: 103.35638278039106, Tokens per sec: 84455.16099984657, Loss: 2.3364739418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2525 , TFLOPS: 102.04230600444455, Tokens per sec: 83381.39503887513, Loss: 2.3698129653930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2526 , TFLOPS: 102.15511931956217, Tokens per sec: 83473.5777027308, Loss: 2.368706703186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2527 , TFLOPS: 101.96318010008294, Tokens per sec: 83316.73922553925, Loss: 2.3557469844818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2528 , TFLOPS: 102.04093346789153, Tokens per sec: 83380.27350392542, Loss: 2.3609166145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2529 , TFLOPS: 102.84162034310238, Tokens per sec: 84034.53536117401, Loss: 2.3644278049468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2530 , TFLOPS: 102.64667013466646, Tokens per sec: 83875.2365273962, Loss: 2.348910331726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2531 , TFLOPS: 101.8825397910916, Tokens per sec: 83250.84595319621, Loss: 2.3545706272125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2532 , TFLOPS: 101.95737158426734, Tokens per sec: 83311.99293774183, Loss: 2.3606386184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2533 , TFLOPS: 103.17725633322647, Tokens per sec: 84308.79216874343, Loss: 2.3478927612304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2534 , TFLOPS: 103.3848814013929, Tokens per sec: 84478.44795668696, Loss: 2.369635820388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2535 , TFLOPS: 101.46270553665235, Tokens per sec: 82907.78857639871, Loss: 2.34938907623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2536 , TFLOPS: 102.07162705087609, Tokens per sec: 83405.3540207052, Loss: 2.378657341003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2537 , TFLOPS: 101.30921680514133, Tokens per sec: 82782.3689827296, Loss: 2.352437734603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2538 , TFLOPS: 102.69684473797503, Tokens per sec: 83916.23548736883, Loss: 2.3845486640930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2539 , TFLOPS: 101.82023128172942, Tokens per sec: 83199.93206623265, Loss: 2.4060218334198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2540 , TFLOPS: 101.46945329458174, Tokens per sec: 82913.30234310556, Loss: 2.354889392852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2541 , TFLOPS: 101.46321714179581, Tokens per sec: 82908.20662213133, Loss: 2.3864431381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2542 , TFLOPS: 102.04828790448228, Tokens per sec: 83386.28300338374, Loss: 2.3420913219451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2543 , TFLOPS: 102.70073911394725, Tokens per sec: 83919.41768222587, Loss: 2.361294984817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2544 , TFLOPS: 100.88896284708467, Tokens per sec: 82438.9686553022, Loss: 2.3479955196380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2545 , TFLOPS: 101.15961041291399, Tokens per sec: 82660.1217484295, Loss: 2.3516108989715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2546 , TFLOPS: 102.80711831496201, Tokens per sec: 84006.3428658193, Loss: 2.3773560523986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2547 , TFLOPS: 102.09941710867831, Tokens per sec: 83428.06199231512, Loss: 2.3290963172912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2548 , TFLOPS: 100.81055892686805, Tokens per sec: 82374.90279379676, Loss: 2.3399498462677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2549 , TFLOPS: 101.63986901285529, Tokens per sec: 83052.55341339773, Loss: 2.3439528942108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2550 , TFLOPS: 102.08559402538856, Tokens per sec: 83416.76679512125, Loss: 2.371958017349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2551 , TFLOPS: 101.51536300550504, Tokens per sec: 82950.8163497221, Loss: 2.3752429485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2552 , TFLOPS: 101.36792664904495, Tokens per sec: 82830.3423075091, Loss: 2.362729787826538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2553 , TFLOPS: 102.78523684631841, Tokens per sec: 83988.46295451149, Loss: 2.35551381111145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2554 , TFLOPS: 101.49254260514559, Tokens per sec: 82932.16921313902, Loss: 2.3649635314941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2555 , TFLOPS: 101.92478728540374, Tokens per sec: 83285.36746834591, Loss: 2.367614269256592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2556 , TFLOPS: 101.99729582558159, Tokens per sec: 83344.6160630611, Loss: 2.3527350425720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2557 , TFLOPS: 102.70494578701272, Tokens per sec: 83922.85506307706, Loss: 2.357144594192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2558 , TFLOPS: 100.90131313641375, Tokens per sec: 82449.06039463774, Loss: 2.336343288421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2559 , TFLOPS: 101.43650269590569, Tokens per sec: 82886.37756070349, Loss: 2.366532564163208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2560 , TFLOPS: 101.55059208067804, Tokens per sec: 82979.60293392302, Loss: 2.3741352558135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2561 , TFLOPS: 101.56953030876556, Tokens per sec: 82995.0778476067, Loss: 2.35306453704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2562 , TFLOPS: 103.36826172576721, Tokens per sec: 84464.8675919047, Loss: 2.3807928562164307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2563 , TFLOPS: 102.13059954538092, Tokens per sec: 83453.54196405216, Loss: 2.3565902709960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2564 , TFLOPS: 102.1578838037275, Tokens per sec: 83475.83663390645, Loss: 2.3574705123901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2565 , TFLOPS: 102.12665276668606, Tokens per sec: 83450.31694958148, Loss: 2.36197829246521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2566 , TFLOPS: 102.78706247355719, Tokens per sec: 83989.95472152413, Loss: 2.3465394973754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2567 , TFLOPS: 102.06524247315483, Tokens per sec: 83400.13701789551, Loss: 2.3413479328155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2568 , TFLOPS: 101.39684862356272, Tokens per sec: 82853.97519740542, Loss: 2.3573312759399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2569 , TFLOPS: 102.0113828649189, Tokens per sec: 83356.12694553628, Loss: 2.353823661804199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2570 , TFLOPS: 101.47637920206488, Tokens per sec: 82918.96168039876, Loss: 2.35064697265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2571 , TFLOPS: 103.33989304857494, Tokens per sec: 84441.68681549617, Loss: 2.3650543689727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2572 , TFLOPS: 102.7619943847926, Tokens per sec: 83969.47094088457, Loss: 2.3471968173980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2573 , TFLOPS: 101.5173185453009, Tokens per sec: 82952.41427162936, Loss: 2.3569090366363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2574 , TFLOPS: 102.26873402507195, Tokens per sec: 83566.41520330556, Loss: 2.3768067359924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2575 , TFLOPS: 101.58038393767742, Tokens per sec: 83003.94662718785, Loss: 2.3656375408172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2576 , TFLOPS: 102.60214258148332, Tokens per sec: 83838.85191744905, Loss: 2.3633182048797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2577 , TFLOPS: 101.50321878310227, Tokens per sec: 82940.89299297669, Loss: 2.3568665981292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2578 , TFLOPS: 101.62369692413955, Tokens per sec: 83039.33878340152, Loss: 2.347332715988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2579 , TFLOPS: 100.6345309911596, Tokens per sec: 82231.06583616718, Loss: 2.3644464015960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2580 , TFLOPS: 101.68362893952805, Tokens per sec: 83088.31077596277, Loss: 2.363581657409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2581 , TFLOPS: 102.16569864165523, Tokens per sec: 83482.22233914901, Loss: 2.372697353363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2582 , TFLOPS: 101.65241846967207, Tokens per sec: 83062.80789761449, Loss: 2.3542306423187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2583 , TFLOPS: 100.39138614655889, Tokens per sec: 82032.38592453882, Loss: 2.3696248531341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2584 , TFLOPS: 102.68185131067574, Tokens per sec: 83903.98397195742, Loss: 2.3385391235351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2585 , TFLOPS: 100.77682927353945, Tokens per sec: 82347.34142578344, Loss: 2.3362157344818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2586 , TFLOPS: 101.64683862238628, Tokens per sec: 83058.24846075923, Loss: 2.333984136581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2587 , TFLOPS: 101.20536517595758, Tokens per sec: 82697.50914314481, Loss: 2.3604977130889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2588 , TFLOPS: 101.03952803203299, Tokens per sec: 82561.99934381616, Loss: 2.3460943698883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2589 , TFLOPS: 102.18071828815957, Tokens per sec: 83494.49527894781, Loss: 2.348416805267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2590 , TFLOPS: 100.40258690658612, Tokens per sec: 82041.53835389028, Loss: 2.3591043949127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2591 , TFLOPS: 102.21658297108125, Tokens per sec: 83523.8012346022, Loss: 2.344789505004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2592 , TFLOPS: 101.48945424517348, Tokens per sec: 82929.64563470434, Loss: 2.386289358139038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2593 , TFLOPS: 102.80687387504835, Tokens per sec: 84006.14312767341, Loss: 2.3776512145996094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2594 , TFLOPS: 101.15807048920098, Tokens per sec: 82658.86343712243, Loss: 2.373124837875366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2595 , TFLOPS: 102.20690874503182, Tokens per sec: 83515.89617546047, Loss: 2.3674721717834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2596 , TFLOPS: 102.35732769913312, Tokens per sec: 83638.80737498499, Loss: 2.3437740802764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2597 , TFLOPS: 100.03364080029597, Tokens per sec: 81740.06299292292, Loss: 2.347928047180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2598 , TFLOPS: 102.07125675527104, Tokens per sec: 83405.05144263405, Loss: 2.3624072074890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2599 , TFLOPS: 102.7788615104854, Tokens per sec: 83983.25350348659, Loss: 2.3621561527252197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2600 , TFLOPS: 101.79132366378484, Tokens per sec: 83176.31090746184, Loss: 2.3661882877349854 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/optimizer.pt +[rank0]:[2024-08-27 09:44:56,098] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0077205770066939294, 'preprocessing_with_comm': 0.0015077499992912635, 'state_converting': 3.3764968549949117, : 3.3873943850048818}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600, took 15.85s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000600 + eval ppl=8.218734741210938, eval loss=2.1064162254333496 +------------------------------------------------------------------ +iteration: 2601 , TFLOPS: 101.76623075540867, Tokens per sec: 83155.80684607875, Loss: 2.3488495349884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2602 , TFLOPS: 98.98609366876047, Tokens per sec: 80884.0852654831, Loss: 2.374356269836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2603 , TFLOPS: 101.60373026707512, Tokens per sec: 83023.02351392669, Loss: 2.3520302772521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2604 , TFLOPS: 100.62344445377789, Tokens per sec: 82222.00674107973, Loss: 2.3517720699310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2605 , TFLOPS: 101.3576338389055, Tokens per sec: 82821.93178738364, Loss: 2.3506197929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2606 , TFLOPS: 101.5619045535557, Tokens per sec: 82988.84664672031, Loss: 2.358673572540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2607 , TFLOPS: 101.27099846326271, Tokens per sec: 82751.13979175295, Loss: 2.367272138595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2608 , TFLOPS: 102.32780702535439, Tokens per sec: 83614.68527251064, Loss: 2.3594179153442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2609 , TFLOPS: 102.85549030075519, Tokens per sec: 84045.8688606166, Loss: 2.3514328002929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2610 , TFLOPS: 102.58776322869848, Tokens per sec: 83827.1021781311, Loss: 2.358154058456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2611 , TFLOPS: 103.33122992048835, Tokens per sec: 84434.60795052786, Loss: 2.345285654067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2612 , TFLOPS: 102.83368081188925, Tokens per sec: 84028.04776583806, Loss: 2.374159812927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2613 , TFLOPS: 103.28021835361125, Tokens per sec: 84392.92508607803, Loss: 2.345168352127075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2614 , TFLOPS: 102.66887999076778, Tokens per sec: 83893.38477254915, Loss: 2.348590850830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2615 , TFLOPS: 101.34621639964922, Tokens per sec: 82812.6023038563, Loss: 2.3694024085998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2616 , TFLOPS: 102.43155017114982, Tokens per sec: 83699.45646752625, Loss: 2.3647468090057373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2617 , TFLOPS: 101.9182528995343, Tokens per sec: 83280.02805344193, Loss: 2.345447540283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2618 , TFLOPS: 102.70316789816927, Tokens per sec: 83921.40230433617, Loss: 2.3720569610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2619 , TFLOPS: 102.7097299600705, Tokens per sec: 83926.76433403815, Loss: 2.3452627658843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2620 , TFLOPS: 102.0396123727522, Tokens per sec: 83379.19400307904, Loss: 2.342989683151245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2621 , TFLOPS: 100.93231435949667, Tokens per sec: 82474.39229206127, Loss: 2.3424723148345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2622 , TFLOPS: 102.01611885738627, Tokens per sec: 83359.99684689665, Loss: 2.3568179607391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2623 , TFLOPS: 103.32114657069243, Tokens per sec: 84426.36858584109, Loss: 2.346144199371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2624 , TFLOPS: 101.73310453669384, Tokens per sec: 83128.73856002212, Loss: 2.3616185188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2625 , TFLOPS: 100.91040142016648, Tokens per sec: 82456.48666524532, Loss: 2.3729360103607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2626 , TFLOPS: 102.63188300406951, Tokens per sec: 83863.15358233082, Loss: 2.368645668029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2627 , TFLOPS: 100.4276835549339, Tokens per sec: 82062.04547130047, Loss: 2.33972430229187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2628 , TFLOPS: 102.11618025136087, Tokens per sec: 83441.75958772273, Loss: 2.331529140472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2629 , TFLOPS: 102.27371034945644, Tokens per sec: 83570.4814860621, Loss: 2.347722291946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2630 , TFLOPS: 100.87853062330473, Tokens per sec: 82430.44421670228, Loss: 2.352806568145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2631 , TFLOPS: 101.47692732202789, Tokens per sec: 82919.4095633304, Loss: 2.334857702255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2632 , TFLOPS: 100.164920519314, Tokens per sec: 81847.33503077315, Loss: 2.341820478439331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2633 , TFLOPS: 101.55606004265019, Tokens per sec: 82984.07094640833, Loss: 2.34480357170105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2634 , TFLOPS: 100.28846771811077, Tokens per sec: 81948.28862729782, Loss: 2.343357801437378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2635 , TFLOPS: 101.54775512888034, Tokens per sec: 82977.28478757948, Loss: 2.3484859466552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2636 , TFLOPS: 100.39593823267911, Tokens per sec: 82036.10555128878, Loss: 2.3575472831726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2637 , TFLOPS: 100.33333689848092, Tokens per sec: 81984.95238961374, Loss: 2.3770666122436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2638 , TFLOPS: 101.43983401200518, Tokens per sec: 82889.09966484409, Loss: 2.3578269481658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2639 , TFLOPS: 100.67126038563178, Tokens per sec: 82261.07836988913, Loss: 2.377497673034668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2640 , TFLOPS: 100.5074896716715, Tokens per sec: 82127.25710367896, Loss: 2.334216356277466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2641 , TFLOPS: 101.69699111736108, Tokens per sec: 83099.22935544321, Loss: 2.380749225616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2642 , TFLOPS: 101.68804962867931, Tokens per sec: 83091.92303486694, Loss: 2.3327620029449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2643 , TFLOPS: 102.40002529966891, Tokens per sec: 83673.69668351681, Loss: 2.354518413543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2644 , TFLOPS: 102.3259816788431, Tokens per sec: 83613.19373488767, Loss: 2.3531599044799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2645 , TFLOPS: 102.90222970162098, Tokens per sec: 84084.06082824324, Loss: 2.3403468132019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2646 , TFLOPS: 102.69243806313847, Tokens per sec: 83912.63467992196, Loss: 2.347923517227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2647 , TFLOPS: 103.40514558532686, Tokens per sec: 84495.0063430259, Loss: 2.339895486831665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2648 , TFLOPS: 103.34555001429098, Tokens per sec: 84446.30926780599, Loss: 2.3531434535980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2649 , TFLOPS: 101.33093327280457, Tokens per sec: 82800.11406748886, Loss: 2.3423540592193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2650 , TFLOPS: 102.6096612877204, Tokens per sec: 83844.99564586408, Loss: 2.3561291694641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2651 , TFLOPS: 103.36728279310019, Tokens per sec: 84464.06768082251, Loss: 2.353151798248291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2652 , TFLOPS: 101.9891016695861, Tokens per sec: 83337.92040726086, Loss: 2.3515095710754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2653 , TFLOPS: 101.32814006755554, Tokens per sec: 82797.83166757636, Loss: 2.371035099029541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2654 , TFLOPS: 102.76637046072707, Tokens per sec: 83973.04674518084, Loss: 2.3572425842285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2655 , TFLOPS: 102.68699750039931, Tokens per sec: 83908.18905605526, Loss: 2.3850784301757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2656 , TFLOPS: 102.02428601109209, Tokens per sec: 83366.67043843091, Loss: 2.3717029094696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2657 , TFLOPS: 102.62495349142075, Tokens per sec: 83857.491299164, Loss: 2.353154420852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2658 , TFLOPS: 101.89502177645772, Tokens per sec: 83261.04530475374, Loss: 2.3608880043029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2659 , TFLOPS: 100.8558104456407, Tokens per sec: 82411.8789746636, Loss: 2.340388774871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2660 , TFLOPS: 102.18659230931783, Tokens per sec: 83499.29509284698, Loss: 2.366616725921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2661 , TFLOPS: 103.28274133993695, Tokens per sec: 84394.98668315221, Loss: 2.3541440963745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2662 , TFLOPS: 101.69971111806437, Tokens per sec: 83101.45193803692, Loss: 2.351238250732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2663 , TFLOPS: 101.57932700737132, Tokens per sec: 83003.08298222693, Loss: 2.352785110473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2664 , TFLOPS: 101.5402414410068, Tokens per sec: 82971.14515979849, Loss: 2.353394031524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2665 , TFLOPS: 102.0866117920779, Tokens per sec: 83417.59843848276, Loss: 2.3512561321258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2666 , TFLOPS: 100.91327669675209, Tokens per sec: 82458.83612775963, Loss: 2.3413331508636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2667 , TFLOPS: 102.1010249956728, Tokens per sec: 83429.37583816903, Loss: 2.3470892906188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2668 , TFLOPS: 101.44201536060498, Tokens per sec: 82890.8821009379, Loss: 2.3664069175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2669 , TFLOPS: 100.739807433143, Tokens per sec: 82317.08992696853, Loss: 2.364140748977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2670 , TFLOPS: 101.07645802048573, Tokens per sec: 82592.17578804336, Loss: 2.355236530303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2671 , TFLOPS: 100.97720407257171, Tokens per sec: 82511.07283217889, Loss: 2.355531692504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2672 , TFLOPS: 100.86748854355542, Tokens per sec: 82421.42144909072, Loss: 2.3547775745391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2673 , TFLOPS: 100.91380813950319, Tokens per sec: 82459.2703833121, Loss: 2.3599939346313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2674 , TFLOPS: 100.97422380539784, Tokens per sec: 82508.63757915222, Loss: 2.36095929145813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2675 , TFLOPS: 101.0417889813308, Tokens per sec: 82563.8468236894, Loss: 2.3593122959136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2676 , TFLOPS: 101.55277202281522, Tokens per sec: 82981.38422076001, Loss: 2.342857837677002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2677 , TFLOPS: 100.57758332691506, Tokens per sec: 82184.53243374969, Loss: 2.3728597164154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2678 , TFLOPS: 100.44449385496088, Tokens per sec: 82075.78160018811, Loss: 2.353376865386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2679 , TFLOPS: 101.81849200683891, Tokens per sec: 83198.51085994669, Loss: 2.370680332183838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2680 , TFLOPS: 101.74043006085307, Tokens per sec: 83134.7244343884, Loss: 2.3457398414611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2681 , TFLOPS: 101.719856627626, Tokens per sec: 83117.91335249128, Loss: 2.3491523265838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2682 , TFLOPS: 102.25313854748117, Tokens per sec: 83553.67173710282, Loss: 2.3489315509796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2683 , TFLOPS: 102.77939312609237, Tokens per sec: 83983.68790028406, Loss: 2.3740785121917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2684 , TFLOPS: 103.41614090548542, Tokens per sec: 84503.99090217217, Loss: 2.3487789630889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2685 , TFLOPS: 102.83044842163146, Tokens per sec: 84025.40649654927, Loss: 2.365396738052368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2686 , TFLOPS: 103.43609885919649, Tokens per sec: 84520.29905991291, Loss: 2.332916498184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2687 , TFLOPS: 101.36624728729542, Tokens per sec: 82828.97005779293, Loss: 2.3583192825317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2688 , TFLOPS: 102.63846757079769, Tokens per sec: 83868.53400130625, Loss: 2.3490371704101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2689 , TFLOPS: 102.72140234360285, Tokens per sec: 83936.30213909631, Loss: 2.3528192043304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2690 , TFLOPS: 102.60291793862076, Tokens per sec: 83839.48548172554, Loss: 2.3687925338745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2691 , TFLOPS: 101.43164439692217, Tokens per sec: 82882.40771954031, Loss: 2.3507988452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2692 , TFLOPS: 102.03311283870863, Tokens per sec: 83373.88306649915, Loss: 2.3522791862487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2693 , TFLOPS: 102.79862275140592, Tokens per sec: 83999.40092214233, Loss: 2.3853092193603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2694 , TFLOPS: 102.01347332617854, Tokens per sec: 83357.83511524477, Loss: 2.36806058883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2695 , TFLOPS: 103.41864446508865, Tokens per sec: 84506.03662517133, Loss: 2.3734138011932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2696 , TFLOPS: 100.28440914284711, Tokens per sec: 81944.97226097286, Loss: 2.353055000305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2697 , TFLOPS: 101.38505636825501, Tokens per sec: 82844.33944203383, Loss: 2.373659610748291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2698 , TFLOPS: 102.154425191505, Tokens per sec: 83473.01051282683, Loss: 2.3481218814849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2699 , TFLOPS: 102.67685348954119, Tokens per sec: 83899.90012365302, Loss: 2.3472700119018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2700 , TFLOPS: 101.4879247582688, Tokens per sec: 82928.39585158204, Loss: 2.3516218662261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2701 , TFLOPS: 101.60095961266458, Tokens per sec: 83020.75954088484, Loss: 2.3391411304473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2702 , TFLOPS: 101.53008231130372, Tokens per sec: 82962.84387339883, Loss: 2.359780788421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2703 , TFLOPS: 101.76158833767393, Tokens per sec: 83152.01340704122, Loss: 2.3381423950195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2704 , TFLOPS: 100.9885217328646, Tokens per sec: 82520.32078374672, Loss: 2.323981761932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2705 , TFLOPS: 101.65727497655027, Tokens per sec: 83066.77627440228, Loss: 2.339073896408081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2706 , TFLOPS: 101.40503605320914, Tokens per sec: 82860.6653569327, Loss: 2.3463962078094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2707 , TFLOPS: 101.45451274118383, Tokens per sec: 82901.09403232008, Loss: 2.349405288696289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2708 , TFLOPS: 100.80491630825806, Tokens per sec: 82370.2920648766, Loss: 2.384913921356201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2709 , TFLOPS: 102.23381057610918, Tokens per sec: 83537.87835414852, Loss: 2.355186700820923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2710 , TFLOPS: 100.24196813454168, Tokens per sec: 81910.29262055775, Loss: 2.361783504486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2711 , TFLOPS: 102.17167637148269, Tokens per sec: 83487.1068960715, Loss: 2.3377041816711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2712 , TFLOPS: 100.52882086110716, Tokens per sec: 82144.6873676805, Loss: 2.3799993991851807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2713 , TFLOPS: 100.1698131905863, Tokens per sec: 81851.33295841831, Loss: 2.341311454772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2714 , TFLOPS: 101.35432209717733, Tokens per sec: 82819.22567796573, Loss: 2.3536126613616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2715 , TFLOPS: 101.0682139527029, Tokens per sec: 82585.43934803683, Loss: 2.3639347553253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2716 , TFLOPS: 100.9337688860929, Tokens per sec: 82475.5808231859, Loss: 2.3569865226745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2717 , TFLOPS: 101.70406367542284, Tokens per sec: 83105.00852469933, Loss: 2.3762266635894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2718 , TFLOPS: 101.67861200561973, Tokens per sec: 83084.21131011893, Loss: 2.3581254482269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2719 , TFLOPS: 101.78725206315106, Tokens per sec: 83172.98389777164, Loss: 2.358318328857422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2720 , TFLOPS: 102.23286066766882, Tokens per sec: 83537.10215950909, Loss: 2.3760104179382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2721 , TFLOPS: 102.8606882351811, Tokens per sec: 84050.11622664274, Loss: 2.3492093086242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2722 , TFLOPS: 102.48408437924948, Tokens per sec: 83742.3835213153, Loss: 2.3741235733032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2723 , TFLOPS: 102.78131406481118, Tokens per sec: 83985.25754876056, Loss: 2.3483614921569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2724 , TFLOPS: 103.3996528770088, Tokens per sec: 84490.51810966432, Loss: 2.3510191440582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2725 , TFLOPS: 101.96918728134882, Tokens per sec: 83321.6478479902, Loss: 2.33097243309021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2726 , TFLOPS: 102.71371182321748, Tokens per sec: 83930.01801691783, Loss: 2.369309902191162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2727 , TFLOPS: 102.94523082895337, Tokens per sec: 84119.19815633404, Loss: 2.3474206924438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2728 , TFLOPS: 102.70746935042415, Tokens per sec: 83924.91713170285, Loss: 2.345399856567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2729 , TFLOPS: 100.6729565334287, Tokens per sec: 82262.46433591674, Loss: 2.3307204246520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2730 , TFLOPS: 102.64177650260844, Tokens per sec: 83871.23781466829, Loss: 2.3476316928863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2731 , TFLOPS: 102.67890982786307, Tokens per sec: 83901.58041063068, Loss: 2.344341278076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2732 , TFLOPS: 102.0488963451312, Tokens per sec: 83386.78017590063, Loss: 2.3468849658966064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2733 , TFLOPS: 102.13591614335513, Tokens per sec: 83457.8862931183, Loss: 2.353910207748413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2734 , TFLOPS: 101.34438283580722, Tokens per sec: 82811.10405164088, Loss: 2.3630433082580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2735 , TFLOPS: 101.36845509644586, Tokens per sec: 82830.77411548381, Loss: 2.357978582382202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2736 , TFLOPS: 102.25630409776211, Tokens per sec: 83556.25838972589, Loss: 2.343858242034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2737 , TFLOPS: 102.76809862081288, Tokens per sec: 83974.45886927375, Loss: 2.3525707721710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2738 , TFLOPS: 101.36498325415343, Tokens per sec: 82827.9371837735, Loss: 2.3286080360412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2739 , TFLOPS: 101.53960856857294, Tokens per sec: 82970.6280234412, Loss: 2.3458189964294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2740 , TFLOPS: 101.59072416135903, Tokens per sec: 83012.39589013913, Loss: 2.353947401046753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2741 , TFLOPS: 101.74202746020681, Tokens per sec: 83136.02971052106, Loss: 2.347177505493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2742 , TFLOPS: 102.14149148018343, Tokens per sec: 83462.44204436264, Loss: 2.3671228885650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2743 , TFLOPS: 100.97397169691537, Tokens per sec: 82508.43157482134, Loss: 2.327226161956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2744 , TFLOPS: 102.03717158742951, Tokens per sec: 83377.19957456047, Loss: 2.3328583240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2745 , TFLOPS: 101.31582260435, Tokens per sec: 82787.76675131141, Loss: 2.3414649963378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2746 , TFLOPS: 100.9498952303051, Tokens per sec: 82488.7580741707, Loss: 2.3441965579986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2747 , TFLOPS: 102.14480941601448, Tokens per sec: 83465.15321514146, Loss: 2.3365111351013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2748 , TFLOPS: 100.11956379777028, Tokens per sec: 81810.2728860139, Loss: 2.342318534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2749 , TFLOPS: 102.12626915799744, Tokens per sec: 83450.00349304703, Loss: 2.3341565132141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2750 , TFLOPS: 98.19493585584738, Tokens per sec: 80237.60985033787, Loss: 2.328669786453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2751 , TFLOPS: 100.97846494310203, Tokens per sec: 82512.10312194692, Loss: 2.354949712753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2752 , TFLOPS: 100.97191419807896, Tokens per sec: 82506.75033955739, Loss: 2.3501687049865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2753 , TFLOPS: 100.64079239049556, Tokens per sec: 82236.18218674736, Loss: 2.3574588298797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2754 , TFLOPS: 101.0571254706777, Tokens per sec: 82576.37866393113, Loss: 2.3450093269348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2755 , TFLOPS: 100.42092508763545, Tokens per sec: 82056.52295369247, Loss: 2.381540060043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2756 , TFLOPS: 101.66707037889462, Tokens per sec: 83074.78034981394, Loss: 2.36295223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2757 , TFLOPS: 102.42218607857941, Tokens per sec: 83691.80482643377, Loss: 2.371615171432495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2758 , TFLOPS: 102.14574585559784, Tokens per sec: 83465.91840403112, Loss: 2.3269174098968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2759 , TFLOPS: 102.13562694377327, Tokens per sec: 83457.64998069529, Loss: 2.3520376682281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2760 , TFLOPS: 103.39060196711327, Tokens per sec: 84483.12237819753, Loss: 2.3464202880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2761 , TFLOPS: 103.31921420999178, Tokens per sec: 84424.78960416921, Loss: 2.3621459007263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2762 , TFLOPS: 102.69589448835181, Tokens per sec: 83915.45901394008, Loss: 2.3635740280151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2763 , TFLOPS: 101.97184682300069, Tokens per sec: 83323.82102793665, Loss: 2.3449485301971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2764 , TFLOPS: 101.30903766291115, Tokens per sec: 82782.2226010018, Loss: 2.3468616008758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2765 , TFLOPS: 103.37534657513301, Tokens per sec: 84470.65680470172, Loss: 2.3488175868988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2766 , TFLOPS: 101.98758218210071, Tokens per sec: 83336.67879492145, Loss: 2.336660861968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2767 , TFLOPS: 101.89810772828102, Tokens per sec: 83263.56691542803, Loss: 2.3451082706451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2768 , TFLOPS: 102.12827328234243, Tokens per sec: 83451.64111464008, Loss: 2.338066816329956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2769 , TFLOPS: 102.0343634845702, Tokens per sec: 83374.90500142713, Loss: 2.372603416442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2770 , TFLOPS: 101.99124660965528, Tokens per sec: 83339.67309300703, Loss: 2.3525514602661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2771 , TFLOPS: 103.31544348723556, Tokens per sec: 84421.70844953805, Loss: 2.3314778804779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2772 , TFLOPS: 100.8955427314586, Tokens per sec: 82444.34524820544, Loss: 2.333425760269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2773 , TFLOPS: 101.07066117845837, Tokens per sec: 82587.43903919881, Loss: 2.34240460395813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2774 , TFLOPS: 102.80015322012852, Tokens per sec: 84000.65150753336, Loss: 2.3489317893981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2775 , TFLOPS: 102.6633104547105, Tokens per sec: 83888.83376126413, Loss: 2.3485326766967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2776 , TFLOPS: 101.45629791581003, Tokens per sec: 82902.55274446162, Loss: 2.330573320388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2777 , TFLOPS: 101.53463861345642, Tokens per sec: 82966.56694517746, Loss: 2.344320058822632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2778 , TFLOPS: 101.92158683957203, Tokens per sec: 83282.75230167001, Loss: 2.361182451248169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2779 , TFLOPS: 102.06182839443146, Tokens per sec: 83397.34728628444, Loss: 2.337376832962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2780 , TFLOPS: 102.03689716065047, Tokens per sec: 83376.97533337494, Loss: 2.355740547180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2781 , TFLOPS: 101.5501816760032, Tokens per sec: 82979.26758169846, Loss: 2.3788468837738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2782 , TFLOPS: 102.15057443724407, Tokens per sec: 83469.863962393, Loss: 2.360734224319458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2783 , TFLOPS: 100.09009104590753, Tokens per sec: 81786.1899417707, Loss: 2.341648817062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2784 , TFLOPS: 101.65799036927223, Tokens per sec: 83067.36084021123, Loss: 2.374542474746704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2785 , TFLOPS: 101.57374629448113, Tokens per sec: 82998.52283806402, Loss: 2.3321197032928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2786 , TFLOPS: 100.94663750282866, Tokens per sec: 82486.09610118835, Loss: 2.351017475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2787 , TFLOPS: 101.04483186752282, Tokens per sec: 82566.33324432776, Loss: 2.337468147277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2788 , TFLOPS: 100.25587894864645, Tokens per sec: 81921.65950485897, Loss: 2.346961736679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2789 , TFLOPS: 101.4034293780928, Tokens per sec: 82859.35250133564, Loss: 2.343034505844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2790 , TFLOPS: 101.01764869416043, Tokens per sec: 82544.12117361622, Loss: 2.3428468704223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2791 , TFLOPS: 100.4009666813515, Tokens per sec: 82040.21442614277, Loss: 2.348966598510742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2792 , TFLOPS: 100.91784275751358, Tokens per sec: 82462.5671735487, Loss: 2.3525919914245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2793 , TFLOPS: 101.67879767911091, Tokens per sec: 83084.36302870823, Loss: 2.33164644241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2794 , TFLOPS: 101.18071459355814, Tokens per sec: 82677.36651770378, Loss: 2.3572473526000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2795 , TFLOPS: 102.47722861833644, Tokens per sec: 83736.78150259027, Loss: 2.3441550731658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2796 , TFLOPS: 101.5796025323162, Tokens per sec: 83003.30812075202, Loss: 2.3384530544281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2797 , TFLOPS: 102.78361189038588, Tokens per sec: 83987.13516118894, Loss: 2.3246965408325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2798 , TFLOPS: 102.83783110079158, Tokens per sec: 84031.43907373807, Loss: 2.35506272315979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2799 , TFLOPS: 102.63212700615249, Tokens per sec: 83863.352962714, Loss: 2.371600866317749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2800 , TFLOPS: 102.77654121366503, Tokens per sec: 83981.35752922492, Loss: 2.3415307998657227 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/optimizer.pt +[rank0]:[2024-08-27 10:37:51,576] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007792788004735485, 'preprocessing_with_comm': 0.0014722129999427125, 'state_converting': 3.3390953179914504, : 3.3500293599936413}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800, took 15.88s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0000800 + eval ppl=8.886791229248047, eval loss=2.184566020965576 +------------------------------------------------------------------ +iteration: 2801 , TFLOPS: 99.63693394251476, Tokens per sec: 81415.90360728686, Loss: 2.359736442565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2802 , TFLOPS: 100.56071388607704, Tokens per sec: 82170.7479793829, Loss: 2.342276096343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2803 , TFLOPS: 100.97892404042939, Tokens per sec: 82512.47826219132, Loss: 2.3521316051483154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2804 , TFLOPS: 101.19618973624547, Tokens per sec: 82690.01165515921, Loss: 2.342036724090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2805 , TFLOPS: 99.80408622004393, Tokens per sec: 81552.48803613835, Loss: 2.3638863563537598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2806 , TFLOPS: 101.8965483362345, Tokens per sec: 83262.2926960445, Loss: 2.31787371635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2807 , TFLOPS: 101.447147684486, Tokens per sec: 82895.07585489885, Loss: 2.349292755126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2808 , TFLOPS: 101.50445224379885, Tokens per sec: 82941.90088546397, Loss: 2.341388702392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2809 , TFLOPS: 101.87128829340337, Tokens per sec: 83241.65206479584, Loss: 2.349954843521118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2810 , TFLOPS: 103.38228715674327, Tokens per sec: 84476.32813260195, Loss: 2.3485543727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2811 , TFLOPS: 102.62312258568397, Tokens per sec: 83855.99521895476, Loss: 2.3541369438171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2812 , TFLOPS: 102.79865371109076, Tokens per sec: 83999.42622009781, Loss: 2.3693981170654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2813 , TFLOPS: 102.70730235030716, Tokens per sec: 83924.7806716083, Loss: 2.362260103225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2814 , TFLOPS: 100.21648582949004, Tokens per sec: 81889.47037312717, Loss: 2.366903781890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2815 , TFLOPS: 102.39468961410918, Tokens per sec: 83669.33675749318, Loss: 2.345942735671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2816 , TFLOPS: 101.4504182913775, Tokens per sec: 82897.74835198281, Loss: 2.3591766357421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2817 , TFLOPS: 102.74897835679852, Tokens per sec: 83958.83520934824, Loss: 2.3593568801879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2818 , TFLOPS: 102.81760466025953, Tokens per sec: 84014.91152850444, Loss: 2.3621597290039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2819 , TFLOPS: 101.60565501957356, Tokens per sec: 83024.59627874069, Loss: 2.359970808029175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2820 , TFLOPS: 102.72918515903247, Tokens per sec: 83942.66167793135, Loss: 2.339554786682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2821 , TFLOPS: 103.23525730299934, Tokens per sec: 84356.18625422263, Loss: 2.349658966064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2822 , TFLOPS: 100.92452412272063, Tokens per sec: 82468.02668905303, Loss: 2.3546810150146484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2823 , TFLOPS: 102.61429744010012, Tokens per sec: 83848.78396531874, Loss: 2.3469700813293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2824 , TFLOPS: 102.18490159990614, Tokens per sec: 83497.91357066395, Loss: 2.3462748527526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2825 , TFLOPS: 102.81243886083442, Tokens per sec: 84010.69042081467, Loss: 2.329695224761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2826 , TFLOPS: 101.48895358875444, Tokens per sec: 82929.23653545631, Loss: 2.36188006401062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2827 , TFLOPS: 101.59829357680835, Tokens per sec: 83018.58105435685, Loss: 2.35860538482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2828 , TFLOPS: 102.79585165510152, Tokens per sec: 83997.13658801766, Loss: 2.3249964714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2829 , TFLOPS: 102.04497591186552, Tokens per sec: 83383.57668895832, Loss: 2.3514513969421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2830 , TFLOPS: 100.24090700620086, Tokens per sec: 81909.42554527459, Loss: 2.346071243286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2831 , TFLOPS: 102.65897744675506, Tokens per sec: 83885.2931489222, Loss: 2.3467113971710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2832 , TFLOPS: 102.83764719027307, Tokens per sec: 84031.28879571913, Loss: 2.345186710357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2833 , TFLOPS: 99.5950112325882, Tokens per sec: 81381.64748181932, Loss: 2.346325159072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2834 , TFLOPS: 102.72102396034768, Tokens per sec: 83935.99295239808, Loss: 2.338685989379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2835 , TFLOPS: 99.90143412114438, Tokens per sec: 81632.03351208546, Loss: 2.359161138534546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2836 , TFLOPS: 102.25519723153174, Tokens per sec: 83555.35394083579, Loss: 2.3417158126831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2837 , TFLOPS: 101.02329832309292, Tokens per sec: 82548.73763085135, Loss: 2.3478527069091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2838 , TFLOPS: 101.5016725960996, Tokens per sec: 82939.62956377442, Loss: 2.35554575920105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2839 , TFLOPS: 100.26338396811424, Tokens per sec: 81927.7920494625, Loss: 2.3711228370666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2840 , TFLOPS: 101.64182104571465, Tokens per sec: 83054.14846969693, Loss: 2.3498454093933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2841 , TFLOPS: 100.16969449229762, Tokens per sec: 81851.23596699107, Loss: 2.343433380126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2842 , TFLOPS: 102.24808540585947, Tokens per sec: 83549.54268500429, Loss: 2.3773813247680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2843 , TFLOPS: 99.9484482283536, Tokens per sec: 81670.4499493369, Loss: 2.3568899631500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2844 , TFLOPS: 101.54981586060472, Tokens per sec: 82978.96866451943, Loss: 2.3581578731536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2845 , TFLOPS: 102.31715079247074, Tokens per sec: 83605.97779030558, Loss: 2.3252499103546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2846 , TFLOPS: 100.30406887266162, Tokens per sec: 81961.03673229077, Loss: 2.3437843322753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2847 , TFLOPS: 102.7106742634879, Tokens per sec: 83927.53594866904, Loss: 2.348970651626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2848 , TFLOPS: 103.46084711204084, Tokens per sec: 84540.52149438874, Loss: 2.33459734916687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2849 , TFLOPS: 102.08608944129918, Tokens per sec: 83417.17161221495, Loss: 2.344419002532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2850 , TFLOPS: 102.24479779757272, Tokens per sec: 83546.85629564262, Loss: 2.353489398956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2851 , TFLOPS: 102.50599286389328, Tokens per sec: 83760.28550809232, Loss: 2.340029716491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2852 , TFLOPS: 100.86871722218737, Tokens per sec: 82422.42543402988, Loss: 2.3420329093933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2853 , TFLOPS: 103.44395411387008, Tokens per sec: 84526.71779071896, Loss: 2.3524887561798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2854 , TFLOPS: 100.1737749174059, Tokens per sec: 81854.57018738652, Loss: 2.3230338096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2855 , TFLOPS: 103.33421373188574, Tokens per sec: 84437.04609963081, Loss: 2.355372190475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2856 , TFLOPS: 102.59336335837286, Tokens per sec: 83831.67818824819, Loss: 2.3735198974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2857 , TFLOPS: 101.57497808993571, Tokens per sec: 82999.52936983919, Loss: 2.3343183994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2858 , TFLOPS: 103.41512672955648, Tokens per sec: 84503.16219291341, Loss: 2.362727403640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2859 , TFLOPS: 102.69154963948412, Tokens per sec: 83911.90872608394, Loss: 2.3412444591522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2860 , TFLOPS: 100.41843159964641, Tokens per sec: 82054.48546046854, Loss: 2.344881057739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2861 , TFLOPS: 102.8219773089163, Tokens per sec: 84018.48453228388, Loss: 2.3497490882873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2862 , TFLOPS: 102.08795791296953, Tokens per sec: 83418.69838851555, Loss: 2.342602014541626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2863 , TFLOPS: 102.72935414294123, Tokens per sec: 83942.79975903324, Loss: 2.33107590675354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2864 , TFLOPS: 100.68719970307342, Tokens per sec: 82274.10279648524, Loss: 2.345597267150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2865 , TFLOPS: 100.74785521755551, Tokens per sec: 82323.6659787804, Loss: 2.3408734798431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2866 , TFLOPS: 103.40072888354221, Tokens per sec: 84491.39734230164, Loss: 2.351954221725464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2867 , TFLOPS: 102.24948480011935, Tokens per sec: 83550.68616607763, Loss: 2.3409714698791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2868 , TFLOPS: 99.97937914438076, Tokens per sec: 81695.72439705531, Loss: 2.3579044342041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2869 , TFLOPS: 103.37112022466123, Tokens per sec: 84467.20334493529, Loss: 2.3334643840789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2870 , TFLOPS: 101.9797514370165, Tokens per sec: 83330.28009153182, Loss: 2.358668088912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2871 , TFLOPS: 99.62011511925799, Tokens per sec: 81402.16051384874, Loss: 2.3475229740142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2872 , TFLOPS: 103.24028205219734, Tokens per sec: 84360.29210614087, Loss: 2.3490488529205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2873 , TFLOPS: 99.97952192071416, Tokens per sec: 81695.84106327272, Loss: 2.3510286808013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2874 , TFLOPS: 103.43104950120306, Tokens per sec: 84516.17309951445, Loss: 2.349928855895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2875 , TFLOPS: 100.60597593520284, Tokens per sec: 82207.73276487239, Loss: 2.3564937114715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2876 , TFLOPS: 102.0581922696259, Tokens per sec: 83394.37611510363, Loss: 2.32879638671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2877 , TFLOPS: 100.40885169375045, Tokens per sec: 82046.65747275224, Loss: 2.359903335571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2878 , TFLOPS: 100.86630982607187, Tokens per sec: 82420.45828869201, Loss: 2.3304591178894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2879 , TFLOPS: 100.16261788142675, Tokens per sec: 81845.45348608031, Loss: 2.3478939533233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2880 , TFLOPS: 102.32859077584764, Tokens per sec: 83615.32569521421, Loss: 2.3574743270874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2881 , TFLOPS: 100.07397332407673, Tokens per sec: 81773.0197363557, Loss: 2.348167896270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2882 , TFLOPS: 101.92404841198062, Tokens per sec: 83284.76371585159, Loss: 2.322326183319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2883 , TFLOPS: 102.09727587445076, Tokens per sec: 83426.31233470752, Loss: 2.3738880157470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2884 , TFLOPS: 100.47300403055732, Tokens per sec: 82099.0779985852, Loss: 2.370314359664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2885 , TFLOPS: 102.63521497876957, Tokens per sec: 83865.8762246309, Loss: 2.3646912574768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2886 , TFLOPS: 103.20055301220711, Tokens per sec: 84327.82848484837, Loss: 2.332458734512329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2887 , TFLOPS: 103.41354500527125, Tokens per sec: 84501.86972528276, Loss: 2.3614659309387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2888 , TFLOPS: 102.04277281584037, Tokens per sec: 83381.77648248342, Loss: 2.3349430561065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2889 , TFLOPS: 103.37027089659065, Tokens per sec: 84466.50933710519, Loss: 2.338362693786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2890 , TFLOPS: 101.42582336357833, Tokens per sec: 82877.65120334835, Loss: 2.335289478302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2891 , TFLOPS: 103.25606705077914, Tokens per sec: 84373.19043482408, Loss: 2.353494882583618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2892 , TFLOPS: 100.15491382125694, Tokens per sec: 81839.15830019541, Loss: 2.3320302963256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2893 , TFLOPS: 103.25138107363388, Tokens per sec: 84369.3614022709, Loss: 2.337970495223999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2894 , TFLOPS: 102.60140067355968, Tokens per sec: 83838.24568538625, Loss: 2.341740131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2895 , TFLOPS: 101.51919013866282, Tokens per sec: 82953.94359874449, Loss: 2.3620171546936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2896 , TFLOPS: 102.17424714422371, Tokens per sec: 83489.20754065576, Loss: 2.3252716064453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2897 , TFLOPS: 101.96026469756028, Tokens per sec: 83314.35697508884, Loss: 2.342879295349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2898 , TFLOPS: 102.05623121061686, Tokens per sec: 83392.7736833052, Loss: 2.325315237045288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2899 , TFLOPS: 102.67186813016389, Tokens per sec: 83895.82645817126, Loss: 2.36306095123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2900 , TFLOPS: 101.3812642556, Tokens per sec: 82841.24080916577, Loss: 2.3451437950134277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2901 , TFLOPS: 102.55135886747084, Tokens per sec: 83797.35523744037, Loss: 2.3224902153015137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2902 , TFLOPS: 101.33173401359626, Tokens per sec: 82800.76837340265, Loss: 2.3294339179992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2903 , TFLOPS: 101.53352803421548, Tokens per sec: 82965.65946229083, Loss: 2.3344852924346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2904 , TFLOPS: 103.34658976622129, Tokens per sec: 84447.15887587385, Loss: 2.32926869392395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2905 , TFLOPS: 101.68886886543326, Tokens per sec: 83092.59245430763, Loss: 2.3513472080230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2906 , TFLOPS: 100.26524436063336, Tokens per sec: 81929.31222408063, Loss: 2.3559014797210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2907 , TFLOPS: 102.69604466242599, Tokens per sec: 83915.58172504192, Loss: 2.3818633556365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2908 , TFLOPS: 102.21947810452454, Tokens per sec: 83526.16692266607, Loss: 2.346544027328491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2909 , TFLOPS: 100.96501035830254, Tokens per sec: 82501.10903435542, Loss: 2.3674893379211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2910 , TFLOPS: 103.4187057682497, Tokens per sec: 84506.08671756231, Loss: 2.3484177589416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2911 , TFLOPS: 100.47638007326941, Tokens per sec: 82101.83664999231, Loss: 2.3531439304351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2912 , TFLOPS: 102.6353896108397, Tokens per sec: 83866.01892099083, Loss: 2.3602135181427 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2913 , TFLOPS: 100.91627290935755, Tokens per sec: 82461.28441020893, Loss: 2.34147310256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2914 , TFLOPS: 102.20548435583231, Tokens per sec: 83514.7322703785, Loss: 2.339688301086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2915 , TFLOPS: 101.60506203732496, Tokens per sec: 83024.11173768064, Loss: 2.3478193283081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2916 , TFLOPS: 100.31640935411065, Tokens per sec: 81971.12045735502, Loss: 2.3695502281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2917 , TFLOPS: 101.49007416284886, Tokens per sec: 82930.152185395, Loss: 2.3716983795166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2918 , TFLOPS: 102.20864206341503, Tokens per sec: 83517.31251453098, Loss: 2.3528549671173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2919 , TFLOPS: 101.23250228835067, Tokens per sec: 82719.68357625263, Loss: 2.328781843185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2920 , TFLOPS: 102.91226405824706, Tokens per sec: 84092.26015934972, Loss: 2.3363795280456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2921 , TFLOPS: 102.21198789130911, Tokens per sec: 83520.04647663249, Loss: 2.3501596450805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2922 , TFLOPS: 101.61654111760963, Tokens per sec: 83033.49158967906, Loss: 2.3609306812286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2923 , TFLOPS: 103.32270114163087, Tokens per sec: 84427.63886577311, Loss: 2.3403818607330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2924 , TFLOPS: 101.22485321975522, Tokens per sec: 82713.43332539873, Loss: 2.3657476902008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2925 , TFLOPS: 103.36541115720419, Tokens per sec: 84462.53831895169, Loss: 2.3532261848449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2926 , TFLOPS: 102.71063829094098, Tokens per sec: 83927.50655457494, Loss: 2.335958957672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2927 , TFLOPS: 102.57537442079669, Tokens per sec: 83816.97896427826, Loss: 2.358827590942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2928 , TFLOPS: 101.36880257022702, Tokens per sec: 82831.05804525517, Loss: 2.3652994632720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2929 , TFLOPS: 102.67321375598434, Tokens per sec: 83896.92600366862, Loss: 2.3479883670806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2930 , TFLOPS: 100.85742482612017, Tokens per sec: 82413.19812651123, Loss: 2.3417279720306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2931 , TFLOPS: 102.71591606572956, Tokens per sec: 83931.8191602167, Loss: 2.3430917263031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2932 , TFLOPS: 102.73257666415859, Tokens per sec: 83945.43296407502, Loss: 2.3450441360473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2933 , TFLOPS: 101.35075770455089, Tokens per sec: 82816.31312099504, Loss: 2.3429250717163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2934 , TFLOPS: 101.35749859721864, Tokens per sec: 82821.82127791987, Loss: 2.3610215187072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2935 , TFLOPS: 101.85145577900344, Tokens per sec: 83225.44640674531, Loss: 2.3412578105926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2936 , TFLOPS: 101.92437746740134, Tokens per sec: 83285.03259550649, Loss: 2.3347225189208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2937 , TFLOPS: 101.63630472888717, Tokens per sec: 83049.64094521456, Loss: 2.321742296218872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2938 , TFLOPS: 101.70631030024049, Tokens per sec: 83106.8442996711, Loss: 2.360806941986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2939 , TFLOPS: 101.40321587111814, Tokens per sec: 82859.17803929049, Loss: 2.3353769779205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2940 , TFLOPS: 101.5904499001805, Tokens per sec: 83012.17178427003, Loss: 2.352205753326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2941 , TFLOPS: 100.91034543298284, Tokens per sec: 82456.44091667632, Loss: 2.358273983001709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2942 , TFLOPS: 103.36711966946159, Tokens per sec: 84463.93438829822, Loss: 2.347559690475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2943 , TFLOPS: 101.00584072090112, Tokens per sec: 82534.47257470177, Loss: 2.3308072090148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2944 , TFLOPS: 100.43641768163744, Tokens per sec: 82069.18235106634, Loss: 2.3696303367614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2945 , TFLOPS: 102.86516876481414, Tokens per sec: 84053.77738274499, Loss: 2.347850799560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2946 , TFLOPS: 101.68025105729028, Tokens per sec: 83085.55062143187, Loss: 2.3641505241394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2947 , TFLOPS: 100.37176836137552, Tokens per sec: 82016.35572725878, Loss: 2.347238540649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2948 , TFLOPS: 102.56481036824849, Tokens per sec: 83808.3468049982, Loss: 2.3276145458221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2949 , TFLOPS: 100.44027608928641, Tokens per sec: 82072.33515528058, Loss: 2.3265058994293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2950 , TFLOPS: 103.40772462745134, Tokens per sec: 84497.11374473626, Loss: 2.3533568382263184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2951 , TFLOPS: 100.89454364261208, Tokens per sec: 82443.5288669901, Loss: 2.3502113819122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2952 , TFLOPS: 102.29446109552342, Tokens per sec: 83587.43745484515, Loss: 2.33453369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2953 , TFLOPS: 101.3797817604362, Tokens per sec: 82840.02942420461, Loss: 2.3504018783569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2954 , TFLOPS: 99.79782701033925, Tokens per sec: 81547.37347476228, Loss: 2.3281311988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2955 , TFLOPS: 102.13511625355622, Tokens per sec: 83457.2326825726, Loss: 2.343855142593384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2956 , TFLOPS: 101.618541718017, Tokens per sec: 83035.12633177197, Loss: 2.3388521671295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2957 , TFLOPS: 101.21781573935874, Tokens per sec: 82707.6828190059, Loss: 2.3444900512695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2958 , TFLOPS: 102.838823947161, Tokens per sec: 84032.25035406467, Loss: 2.335968255996704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2959 , TFLOPS: 102.13545086374162, Tokens per sec: 83457.50610116874, Loss: 2.3214058876037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2960 , TFLOPS: 101.4829734078751, Tokens per sec: 82924.34997571625, Loss: 2.349043846130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2961 , TFLOPS: 103.39607449117037, Tokens per sec: 84487.59411847989, Loss: 2.3534207344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2962 , TFLOPS: 101.36747418942394, Tokens per sec: 82829.97259110541, Loss: 2.3787617683410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2963 , TFLOPS: 103.38544204486224, Tokens per sec: 84478.90607289801, Loss: 2.358846426010132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2964 , TFLOPS: 102.72971460456426, Tokens per sec: 83943.09430150453, Loss: 2.3458542823791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2965 , TFLOPS: 103.40879123823821, Tokens per sec: 84497.98529986721, Loss: 2.328634262084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2966 , TFLOPS: 100.74522601465983, Tokens per sec: 82321.51758941264, Loss: 2.3502626419067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2967 , TFLOPS: 103.38796924934047, Tokens per sec: 84480.97111673318, Loss: 2.3496921062469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2968 , TFLOPS: 100.82990486162174, Tokens per sec: 82390.71085509266, Loss: 2.354747772216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2969 , TFLOPS: 102.79899312415023, Tokens per sec: 83999.70356324596, Loss: 2.3305530548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2970 , TFLOPS: 103.36120239332955, Tokens per sec: 84459.0992296465, Loss: 2.335911273956299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2971 , TFLOPS: 100.61962704557558, Tokens per sec: 82218.8874385688, Loss: 2.341200113296509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2972 , TFLOPS: 102.09387768619635, Tokens per sec: 83423.5355876077, Loss: 2.3600776195526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2973 , TFLOPS: 102.09585624017897, Tokens per sec: 83425.1523149994, Loss: 2.3359172344207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2974 , TFLOPS: 102.7325080867989, Tokens per sec: 83945.37692774908, Loss: 2.3331005573272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2975 , TFLOPS: 101.5742595405928, Tokens per sec: 82998.942224674, Loss: 2.3183813095092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2976 , TFLOPS: 101.38405104397907, Tokens per sec: 82843.51796568862, Loss: 2.325106620788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2977 , TFLOPS: 101.27254680311216, Tokens per sec: 82752.40498010174, Loss: 2.3526601791381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2978 , TFLOPS: 102.07048936744773, Tokens per sec: 83404.42439028932, Loss: 2.3621230125427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2979 , TFLOPS: 100.4078122017846, Tokens per sec: 82045.80807710803, Loss: 2.3575122356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2980 , TFLOPS: 102.16793224046333, Tokens per sec: 83484.04747023307, Loss: 2.33754563331604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2981 , TFLOPS: 101.63209817078145, Tokens per sec: 83046.20365829993, Loss: 2.3236706256866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2982 , TFLOPS: 100.38359370483414, Tokens per sec: 82026.01851981024, Loss: 2.3266959190368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2983 , TFLOPS: 102.69968960162485, Tokens per sec: 83918.5600986904, Loss: 2.3593034744262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2984 , TFLOPS: 102.37274080735318, Tokens per sec: 83651.4018224803, Loss: 2.334829807281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2985 , TFLOPS: 100.20900196343771, Tokens per sec: 81883.35511352395, Loss: 2.3358824253082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2986 , TFLOPS: 103.40430382238239, Tokens per sec: 84494.31851685506, Loss: 2.3525590896606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2987 , TFLOPS: 101.0336977758027, Tokens per sec: 82557.23529136623, Loss: 2.3521652221679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2988 , TFLOPS: 103.36510230328076, Tokens per sec: 84462.28594646024, Loss: 2.3429675102233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2989 , TFLOPS: 100.6915480675606, Tokens per sec: 82277.65595704457, Loss: 2.335062265396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2990 , TFLOPS: 102.18155193955825, Tokens per sec: 83495.17647696577, Loss: 2.3313815593719482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2991 , TFLOPS: 102.11108739371116, Tokens per sec: 83437.59808263533, Loss: 2.3407626152038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2992 , TFLOPS: 100.40918808003347, Tokens per sec: 82046.93234264344, Loss: 2.3528971672058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2993 , TFLOPS: 102.67888600543526, Tokens per sec: 83901.56094471167, Loss: 2.3489573001861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2994 , TFLOPS: 101.60994053858516, Tokens per sec: 83028.09808664415, Loss: 2.3501474857330322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2995 , TFLOPS: 100.78217547224016, Tokens per sec: 82351.70994236517, Loss: 2.34047532081604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2996 , TFLOPS: 103.46883426537744, Tokens per sec: 84547.0480029877, Loss: 2.353400230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2997 , TFLOPS: 102.93504881635378, Tokens per sec: 84110.8781717306, Loss: 2.3470146656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2998 , TFLOPS: 102.03536602314807, Tokens per sec: 83375.72420150675, Loss: 2.3381924629211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2999 , TFLOPS: 103.3087103445649, Tokens per sec: 84416.20662533534, Loss: 2.3448374271392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3000 , TFLOPS: 98.3724986805608, Tokens per sec: 80382.70100528497, Loss: 2.3367438316345215 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/model.pt +[rank0]:[2024-08-27 11:30:45,637] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007659958995645866, 'preprocessing_with_comm': 0.0016645989962853491, 'state_converting': 3.358144428988453, : 3.369088497987832}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000, took 15.84s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001000 + eval ppl=8.52076530456543, eval loss=2.1425061225891113 +------------------------------------------------------------------ +iteration: 3001 , TFLOPS: 101.45149954052863, Tokens per sec: 82898.63186849802, Loss: 2.3423852920532227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3002 , TFLOPS: 100.07545768873585, Tokens per sec: 81774.23264892961, Loss: 2.36191725730896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3003 , TFLOPS: 101.21768459674895, Tokens per sec: 82707.57565900355, Loss: 2.331111431121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3004 , TFLOPS: 100.91217357383015, Tokens per sec: 82457.9347376232, Loss: 2.3507463932037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3005 , TFLOPS: 99.9538846546386, Tokens per sec: 81674.89219320043, Loss: 2.353651523590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3006 , TFLOPS: 102.07484155344996, Tokens per sec: 83407.98067350696, Loss: 2.342268705368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3007 , TFLOPS: 101.61277881826484, Tokens per sec: 83030.41731803425, Loss: 2.332548141479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3008 , TFLOPS: 102.03324323695198, Tokens per sec: 83373.98961826053, Loss: 2.3492870330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3009 , TFLOPS: 103.34495715688134, Tokens per sec: 84445.82482875507, Loss: 2.3408732414245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3010 , TFLOPS: 103.30908189608772, Tokens per sec: 84416.51022964923, Loss: 2.3414125442504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3011 , TFLOPS: 103.3400434094553, Tokens per sec: 84441.80967924214, Loss: 2.3429415225982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3012 , TFLOPS: 102.65229183826246, Tokens per sec: 83879.83016612001, Loss: 2.365931272506714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3013 , TFLOPS: 101.90243577087786, Tokens per sec: 83267.10347044763, Loss: 2.3470959663391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3014 , TFLOPS: 102.94540220099375, Tokens per sec: 84119.33818883974, Loss: 2.3105809688568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3015 , TFLOPS: 102.80636967449084, Tokens per sec: 84005.73113241857, Loss: 2.347947835922241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3016 , TFLOPS: 101.23223201953226, Tokens per sec: 82719.46273264376, Loss: 2.3328399658203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3017 , TFLOPS: 102.56117408603885, Tokens per sec: 83805.37550519849, Loss: 2.343151569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3018 , TFLOPS: 103.4609266451033, Tokens per sec: 84540.58648290145, Loss: 2.339157819747925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3019 , TFLOPS: 102.05065484490717, Tokens per sec: 83388.21709133539, Loss: 2.327571392059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3020 , TFLOPS: 102.77589088243981, Tokens per sec: 83980.82612683978, Loss: 2.330075263977051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3021 , TFLOPS: 102.64050115904817, Tokens per sec: 83870.19569861493, Loss: 2.3190011978149414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3022 , TFLOPS: 101.88192240725562, Tokens per sec: 83250.34147297093, Loss: 2.3491010665893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3023 , TFLOPS: 100.65254929711632, Tokens per sec: 82245.7890577969, Loss: 2.344506025314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3024 , TFLOPS: 103.34893073154294, Tokens per sec: 84449.07173889998, Loss: 2.3512697219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3025 , TFLOPS: 102.16191836516438, Tokens per sec: 83479.13337791536, Loss: 2.342496871948242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3026 , TFLOPS: 102.69514833378686, Tokens per sec: 83914.84931183734, Loss: 2.3418781757354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3027 , TFLOPS: 100.430009678408, Tokens per sec: 82063.9462066709, Loss: 2.3322808742523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3028 , TFLOPS: 103.45477601094329, Tokens per sec: 84535.5606413979, Loss: 2.348461151123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3029 , TFLOPS: 102.22405896685028, Tokens per sec: 83529.91006319431, Loss: 2.3351213932037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3030 , TFLOPS: 100.78365325014059, Tokens per sec: 82352.91747272901, Loss: 2.340975284576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3031 , TFLOPS: 101.91811415856046, Tokens per sec: 83279.91468462069, Loss: 2.3411762714385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3032 , TFLOPS: 101.47800936086828, Tokens per sec: 82920.29372512104, Loss: 2.3458337783813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3033 , TFLOPS: 101.41109973372798, Tokens per sec: 82865.62014637767, Loss: 2.3385658264160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3034 , TFLOPS: 100.5268953577007, Tokens per sec: 82143.11398928022, Loss: 2.348435401916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3035 , TFLOPS: 102.51968087526495, Tokens per sec: 83771.47033454488, Loss: 2.3483190536499023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3036 , TFLOPS: 100.28589463306483, Tokens per sec: 81946.18609326969, Loss: 2.359722137451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3037 , TFLOPS: 102.01469657914974, Tokens per sec: 83358.83466673689, Loss: 2.362022876739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3038 , TFLOPS: 100.87382269430769, Tokens per sec: 82426.59724672594, Loss: 2.3466267585754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3039 , TFLOPS: 102.62661208349817, Tokens per sec: 83858.84657744641, Loss: 2.3408100605010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3040 , TFLOPS: 100.36235172339592, Tokens per sec: 82008.66114995982, Loss: 2.3380491733551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3041 , TFLOPS: 101.78022544251034, Tokens per sec: 83167.24226516481, Loss: 2.3353331089019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3042 , TFLOPS: 101.4778159860845, Tokens per sec: 82920.13571360729, Loss: 2.3294832706451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3043 , TFLOPS: 100.63097099121012, Tokens per sec: 82228.15686856591, Loss: 2.333747148513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3044 , TFLOPS: 100.61601253353872, Tokens per sec: 82215.93392773767, Loss: 2.3509888648986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3045 , TFLOPS: 102.36733995247313, Tokens per sec: 83646.98864492745, Loss: 2.327004909515381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3046 , TFLOPS: 101.54945234072025, Tokens per sec: 82978.671623064, Loss: 2.337606906890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3047 , TFLOPS: 102.6887806809868, Tokens per sec: 83909.64613881645, Loss: 2.3518126010894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3048 , TFLOPS: 103.42995466120257, Tokens per sec: 84515.27847756636, Loss: 2.343937873840332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3049 , TFLOPS: 102.69063389635151, Tokens per sec: 83911.16044879696, Loss: 2.3469061851501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3050 , TFLOPS: 102.68789469806507, Tokens per sec: 83908.92217936386, Loss: 2.3555736541748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3051 , TFLOPS: 102.00076213095568, Tokens per sec: 83347.44847041291, Loss: 2.3283767700195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3052 , TFLOPS: 103.36494939413666, Tokens per sec: 84462.16100046237, Loss: 2.331369400024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3053 , TFLOPS: 102.66801088146049, Tokens per sec: 83892.6746009613, Loss: 2.3257346153259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3054 , TFLOPS: 102.0148850441348, Tokens per sec: 83358.98866632768, Loss: 2.3184125423431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3055 , TFLOPS: 101.44381995135868, Tokens per sec: 82892.35667849705, Loss: 2.348600149154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3056 , TFLOPS: 103.43466044629106, Tokens per sec: 84519.12369569986, Loss: 2.3410558700561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3057 , TFLOPS: 103.39283304087425, Tokens per sec: 84484.94544600057, Loss: 2.3403983116149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3058 , TFLOPS: 101.39387157906954, Tokens per sec: 82851.5425777141, Loss: 2.3359198570251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3059 , TFLOPS: 102.12007007166284, Tokens per sec: 83444.93806002435, Loss: 2.3236207962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3060 , TFLOPS: 101.92449690568543, Tokens per sec: 83285.13019160304, Loss: 2.337941884994507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3061 , TFLOPS: 101.47526750473992, Tokens per sec: 82918.0532838969, Loss: 2.3366191387176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3062 , TFLOPS: 103.42250544314102, Tokens per sec: 84509.19152972834, Loss: 2.338805913925171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3063 , TFLOPS: 102.18983542882211, Tokens per sec: 83501.94512927925, Loss: 2.3308160305023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3064 , TFLOPS: 100.80603013131586, Tokens per sec: 82371.20219836949, Loss: 2.340557098388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3065 , TFLOPS: 102.14371936820082, Tokens per sec: 83464.2625090126, Loss: 2.3423383235931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3066 , TFLOPS: 102.187764885935, Tokens per sec: 83500.25323538568, Loss: 2.3342981338500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3067 , TFLOPS: 102.11413735058635, Tokens per sec: 83440.09028091092, Loss: 2.3486328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3068 , TFLOPS: 100.8395082168401, Tokens per sec: 82398.55800385393, Loss: 2.349675178527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3069 , TFLOPS: 101.91328983593381, Tokens per sec: 83275.97260641336, Loss: 2.3408889770507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3070 , TFLOPS: 101.44635075258525, Tokens per sec: 82894.4246613279, Loss: 2.3523378372192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3071 , TFLOPS: 101.36519979974254, Tokens per sec: 82828.11412874874, Loss: 2.3482511043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3072 , TFLOPS: 101.05475309797133, Tokens per sec: 82574.44013712244, Loss: 2.3310189247131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3073 , TFLOPS: 100.88352599160262, Tokens per sec: 82434.52606073071, Loss: 2.349623203277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3074 , TFLOPS: 101.52406708597125, Tokens per sec: 82957.928677934, Loss: 2.3296828269958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3075 , TFLOPS: 101.54918533505177, Tokens per sec: 82978.45344585895, Loss: 2.3428738117218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3076 , TFLOPS: 101.51540469233846, Tokens per sec: 82950.85041310679, Loss: 2.31496524810791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3077 , TFLOPS: 102.62589329791246, Tokens per sec: 83858.25923924113, Loss: 2.3347744941711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3078 , TFLOPS: 102.16506580788914, Tokens per sec: 83481.70523438815, Loss: 2.349687099456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3079 , TFLOPS: 101.98638749287257, Tokens per sec: 83335.70258359911, Loss: 2.345069646835327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3080 , TFLOPS: 102.1236059268252, Tokens per sec: 83447.82729829852, Loss: 2.3313145637512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3081 , TFLOPS: 100.72136659597916, Tokens per sec: 82302.02144421259, Loss: 2.3246243000030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3082 , TFLOPS: 102.03344036838295, Tokens per sec: 83374.1506994274, Loss: 2.345120668411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3083 , TFLOPS: 100.91871392417762, Tokens per sec: 82463.27902625572, Loss: 2.3371686935424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3084 , TFLOPS: 102.121721256562, Tokens per sec: 83446.28728571073, Loss: 2.344576358795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3085 , TFLOPS: 102.60993609592154, Tokens per sec: 83845.22019871944, Loss: 2.3548638820648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3086 , TFLOPS: 103.3502780312706, Tokens per sec: 84450.17265219003, Loss: 2.341550827026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3087 , TFLOPS: 102.15112911728944, Tokens per sec: 83470.31720573682, Loss: 2.355137586593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3088 , TFLOPS: 102.74413395235405, Tokens per sec: 83954.87672177059, Loss: 2.33333420753479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3089 , TFLOPS: 102.58050892704894, Tokens per sec: 83821.17450151092, Loss: 2.354665517807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3090 , TFLOPS: 102.0594502900388, Tokens per sec: 83395.40407596734, Loss: 2.3397603034973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3091 , TFLOPS: 102.6511174562192, Tokens per sec: 83878.87054832118, Loss: 2.322159767150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3092 , TFLOPS: 102.05991566251983, Tokens per sec: 83395.78434380135, Loss: 2.3296117782592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3093 , TFLOPS: 101.96742793582514, Tokens per sec: 83320.21024147322, Loss: 2.3527257442474365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3094 , TFLOPS: 103.37410570147463, Tokens per sec: 84469.64285489255, Loss: 2.3367421627044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3095 , TFLOPS: 103.22454874415307, Tokens per sec: 84347.43601512651, Loss: 2.3332011699676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3096 , TFLOPS: 102.1887969892699, Tokens per sec: 83501.09659358933, Loss: 2.3115696907043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3097 , TFLOPS: 102.7045485258706, Tokens per sec: 83922.53045077145, Loss: 2.335184097290039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3098 , TFLOPS: 101.30695647053791, Tokens per sec: 82780.52200513898, Loss: 2.3436660766601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3099 , TFLOPS: 101.89081088444144, Tokens per sec: 83257.60447648924, Loss: 2.339933156967163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3100 , TFLOPS: 102.72452962914193, Tokens per sec: 83938.85752461379, Loss: 2.3535284996032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3101 , TFLOPS: 102.19261913909827, Tokens per sec: 83504.21977060498, Loss: 2.3573410511016846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3102 , TFLOPS: 102.1949311169548, Tokens per sec: 83506.10894722716, Loss: 2.347667932510376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3103 , TFLOPS: 101.61344584980885, Tokens per sec: 83030.96236667977, Loss: 2.369776487350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3104 , TFLOPS: 103.39416071236597, Tokens per sec: 84486.03032055376, Loss: 2.3471574783325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3105 , TFLOPS: 101.59315964704224, Tokens per sec: 83014.38598818587, Loss: 2.327317476272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3106 , TFLOPS: 100.70299424746854, Tokens per sec: 82287.00892529824, Loss: 2.333632230758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3107 , TFLOPS: 102.64269550197135, Tokens per sec: 83871.98875270487, Loss: 2.345097541809082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3108 , TFLOPS: 102.73913990118533, Tokens per sec: 83950.79595400231, Loss: 2.3268823623657227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3109 , TFLOPS: 101.6145515028216, Tokens per sec: 83031.86582421852, Loss: 2.3462729454040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3110 , TFLOPS: 101.6061932977127, Tokens per sec: 83025.0361196647, Loss: 2.3545844554901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3111 , TFLOPS: 102.60705449306619, Tokens per sec: 83842.86556684728, Loss: 2.3653736114501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3112 , TFLOPS: 101.59767672724128, Tokens per sec: 83018.07701069648, Loss: 2.3512933254241943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3113 , TFLOPS: 102.7395500782442, Tokens per sec: 83951.13112023602, Loss: 2.3490631580352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3114 , TFLOPS: 100.81546524908065, Tokens per sec: 82378.91187597663, Loss: 2.3221654891967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3115 , TFLOPS: 101.96967083759351, Tokens per sec: 83322.04297424559, Loss: 2.331939458847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3116 , TFLOPS: 102.23764391234215, Tokens per sec: 83541.01067186333, Loss: 2.342364549636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3117 , TFLOPS: 101.92746689933638, Tokens per sec: 83287.55704986968, Loss: 2.3280115127563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3118 , TFLOPS: 102.07737095820148, Tokens per sec: 83410.04751524211, Loss: 2.343940019607544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3119 , TFLOPS: 101.09255220996417, Tokens per sec: 82605.3267645675, Loss: 2.3642852306365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3120 , TFLOPS: 101.3050115643603, Tokens per sec: 82778.9327722349, Loss: 2.35213565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3121 , TFLOPS: 101.6324301686673, Tokens per sec: 83046.47494231885, Loss: 2.3434770107269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3122 , TFLOPS: 101.63885803686487, Tokens per sec: 83051.72731889113, Loss: 2.3384580612182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3123 , TFLOPS: 102.59334133904662, Tokens per sec: 83831.6601956899, Loss: 2.3359756469726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3124 , TFLOPS: 103.23656410382009, Tokens per sec: 84357.2540748133, Loss: 2.3415961265563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3125 , TFLOPS: 101.61928816966135, Tokens per sec: 83035.73627662593, Loss: 2.3544132709503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3126 , TFLOPS: 102.62967652824992, Tokens per sec: 83861.3506141389, Loss: 2.3283891677856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3127 , TFLOPS: 102.57789377024619, Tokens per sec: 83819.03758956728, Loss: 2.3683509826660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3128 , TFLOPS: 102.19488117545893, Tokens per sec: 83506.06813874528, Loss: 2.3297061920166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3129 , TFLOPS: 102.11537447496681, Tokens per sec: 83441.10116708861, Loss: 2.351893424987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3130 , TFLOPS: 100.87593416804917, Tokens per sec: 82428.32258627427, Loss: 2.3295724391937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3131 , TFLOPS: 102.18696584206327, Tokens per sec: 83499.60031606887, Loss: 2.3400485515594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3132 , TFLOPS: 101.99463915478594, Tokens per sec: 83342.4452289652, Loss: 2.322185516357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3133 , TFLOPS: 103.38877617471867, Tokens per sec: 84481.63047623182, Loss: 2.3475418090820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3134 , TFLOPS: 101.53540640221655, Tokens per sec: 82967.194325138, Loss: 2.3445310592651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3135 , TFLOPS: 102.12537384220458, Tokens per sec: 83449.27190746521, Loss: 2.3128714561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3136 , TFLOPS: 101.411572932006, Tokens per sec: 82866.00680887184, Loss: 2.312992572784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3137 , TFLOPS: 102.23746409524138, Tokens per sec: 83540.86373868138, Loss: 2.3171334266662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3138 , TFLOPS: 102.1529499075678, Tokens per sec: 83471.80502034457, Loss: 2.3311612606048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3139 , TFLOPS: 102.21082605423247, Tokens per sec: 83519.09710964884, Loss: 2.337082862854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3140 , TFLOPS: 102.01587646628187, Tokens per sec: 83359.79878288555, Loss: 2.3639872074127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3141 , TFLOPS: 100.85913942979317, Tokens per sec: 82414.59917331027, Loss: 2.337574005126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3142 , TFLOPS: 103.30252114878337, Tokens per sec: 84411.1492741382, Loss: 2.341597557067871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3143 , TFLOPS: 101.81835242677313, Tokens per sec: 83198.39680548183, Loss: 2.327763557434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3144 , TFLOPS: 101.61726616915327, Tokens per sec: 83034.08404795987, Loss: 2.354437828063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3145 , TFLOPS: 102.66164169015015, Tokens per sec: 83887.47017078404, Loss: 2.355360269546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3146 , TFLOPS: 102.07223994886763, Tokens per sec: 83405.85483543148, Loss: 2.342400550842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3147 , TFLOPS: 100.55692730208821, Tokens per sec: 82167.65386413029, Loss: 2.3520665168762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3148 , TFLOPS: 102.36411752131103, Tokens per sec: 83644.35551347212, Loss: 2.3269128799438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3149 , TFLOPS: 102.52382003629069, Tokens per sec: 83774.85254957048, Loss: 2.2971434593200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3150 , TFLOPS: 101.41634514282, Tokens per sec: 82869.90630517561, Loss: 2.3156182765960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3151 , TFLOPS: 102.06451995510675, Tokens per sec: 83399.54662979914, Loss: 2.3562874794006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3152 , TFLOPS: 101.52959042179424, Tokens per sec: 82962.44193781826, Loss: 2.3541290760040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3153 , TFLOPS: 102.01232346395138, Tokens per sec: 83356.89553321888, Loss: 2.349726676940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3154 , TFLOPS: 101.59138254714559, Tokens per sec: 83012.93387411366, Loss: 2.344585657119751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3155 , TFLOPS: 102.62304178388798, Tokens per sec: 83855.92919372725, Loss: 2.343665838241577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3156 , TFLOPS: 101.25203078880416, Tokens per sec: 82735.64081668148, Loss: 2.3402812480926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3157 , TFLOPS: 101.10670625506496, Tokens per sec: 82616.89239917688, Loss: 2.315951347351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3158 , TFLOPS: 102.01024539626987, Tokens per sec: 83355.1974906221, Loss: 2.3201828002929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3159 , TFLOPS: 101.02814126798897, Tokens per sec: 82552.6949257944, Loss: 2.3287789821624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3160 , TFLOPS: 101.45719815576054, Tokens per sec: 82903.28835369942, Loss: 2.3296866416931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3161 , TFLOPS: 103.40340586925034, Tokens per sec: 84493.5847762355, Loss: 2.3536980152130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3162 , TFLOPS: 102.12744936035668, Tokens per sec: 83450.96786677587, Loss: 2.315080404281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3163 , TFLOPS: 102.70632793402963, Tokens per sec: 83923.98445098422, Loss: 2.3280739784240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3164 , TFLOPS: 101.46613933304468, Tokens per sec: 82910.59441982467, Loss: 2.339329957962036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3165 , TFLOPS: 103.29722916111147, Tokens per sec: 84406.8250547836, Loss: 2.337007522583008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3166 , TFLOPS: 101.37254375199421, Tokens per sec: 82834.11506118374, Loss: 2.325519561767578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3167 , TFLOPS: 102.12421009549779, Tokens per sec: 83448.32098007358, Loss: 2.330867052078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3168 , TFLOPS: 101.25768957091171, Tokens per sec: 82740.26475321158, Loss: 2.343494415283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3169 , TFLOPS: 101.67744476822764, Tokens per sec: 83083.25753039849, Loss: 2.3252604007720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3170 , TFLOPS: 102.07629421847774, Tokens per sec: 83409.1676834961, Loss: 2.3202404975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3171 , TFLOPS: 103.2017621613812, Tokens per sec: 84328.81651176479, Loss: 2.3299713134765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3172 , TFLOPS: 101.58866302882625, Tokens per sec: 83010.71168568832, Loss: 2.3536646366119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3173 , TFLOPS: 102.05397381587633, Tokens per sec: 83390.92910795221, Loss: 2.342963218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3174 , TFLOPS: 101.397442955912, Tokens per sec: 82854.46084166749, Loss: 2.3020081520080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3175 , TFLOPS: 102.69199114679019, Tokens per sec: 83912.26949306914, Loss: 2.3518221378326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3176 , TFLOPS: 102.74730774631776, Tokens per sec: 83957.47011051906, Loss: 2.325802803039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3177 , TFLOPS: 102.0997651774039, Tokens per sec: 83428.3464082309, Loss: 2.344090223312378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3178 , TFLOPS: 101.39128212910673, Tokens per sec: 82849.42667149112, Loss: 2.3456149101257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3179 , TFLOPS: 101.15249472994093, Tokens per sec: 82654.30734069824, Loss: 2.331329822540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3180 , TFLOPS: 103.43918279709034, Tokens per sec: 84522.8190249536, Loss: 2.339873790740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3181 , TFLOPS: 101.68786623974202, Tokens per sec: 83091.77318304541, Loss: 2.3562893867492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3182 , TFLOPS: 100.77014996808187, Tokens per sec: 82341.88359335343, Loss: 2.352078437805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3183 , TFLOPS: 101.99717218000858, Tokens per sec: 83344.51502908047, Loss: 2.3314225673675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3184 , TFLOPS: 102.75240416567046, Tokens per sec: 83961.63452596567, Loss: 2.359501838684082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3185 , TFLOPS: 101.3861579464981, Tokens per sec: 82845.23956997342, Loss: 2.3351309299468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3186 , TFLOPS: 101.68356806629238, Tokens per sec: 83088.2610348749, Loss: 2.3592166900634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3187 , TFLOPS: 102.49935034889805, Tokens per sec: 83754.85773809617, Loss: 2.344487190246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3188 , TFLOPS: 101.33656078969157, Tokens per sec: 82804.7124564024, Loss: 2.3305914402008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3189 , TFLOPS: 101.14497187343116, Tokens per sec: 82648.16022098856, Loss: 2.350123643875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3190 , TFLOPS: 102.22523242952414, Tokens per sec: 83530.86892975273, Loss: 2.3260035514831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3191 , TFLOPS: 101.37539512299463, Tokens per sec: 82836.444989829, Loss: 2.3329293727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3192 , TFLOPS: 102.1150835472462, Tokens per sec: 83440.86344255896, Loss: 2.330941915512085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3193 , TFLOPS: 101.96564013364787, Tokens per sec: 83318.74938229205, Loss: 2.3341565132141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3194 , TFLOPS: 101.51445475661305, Tokens per sec: 82950.07419617205, Loss: 2.338963270187378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3195 , TFLOPS: 101.16944245602696, Tokens per sec: 82668.15576395641, Loss: 2.3454251289367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3196 , TFLOPS: 101.46926324842256, Tokens per sec: 82913.14705149668, Loss: 2.3099684715270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3197 , TFLOPS: 101.01409947672893, Tokens per sec: 82541.22101668795, Loss: 2.344935894012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3198 , TFLOPS: 101.56882534199289, Tokens per sec: 82994.50180110929, Loss: 2.3442344665527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3199 , TFLOPS: 102.69755136731683, Tokens per sec: 83916.81289239427, Loss: 2.3560235500335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3200 , TFLOPS: 102.56118083386741, Tokens per sec: 83805.38101902291, Loss: 2.3590846061706543 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/optimizer.pt +[rank0]:[2024-08-27 12:23:35,012] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007642738011782058, 'preprocessing_with_comm': 0.0015160930051933974, 'state_converting': 3.3099625470058527, : 3.320729832004872}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200, took 15.79s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001200 + eval ppl=7.888346195220947, eval loss=2.0653865337371826 +------------------------------------------------------------------ +iteration: 3201 , TFLOPS: 101.28671654054173, Tokens per sec: 82763.98343731694, Loss: 2.3599154949188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3202 , TFLOPS: 100.30585241573942, Tokens per sec: 81962.49411125212, Loss: 2.3400745391845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3203 , TFLOPS: 101.62874208286183, Tokens per sec: 83043.46131246742, Loss: 2.3460593223571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3204 , TFLOPS: 100.95729638404951, Tokens per sec: 82494.80574741792, Loss: 2.31740403175354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3205 , TFLOPS: 100.14256429556421, Tokens per sec: 81829.0671848468, Loss: 2.3162901401519775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3206 , TFLOPS: 102.59909628893271, Tokens per sec: 83836.36271338741, Loss: 2.3378565311431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3207 , TFLOPS: 101.13212608172917, Tokens per sec: 82637.66359390876, Loss: 2.338265895843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3208 , TFLOPS: 101.41329584087325, Tokens per sec: 82867.4146420589, Loss: 2.3328020572662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3209 , TFLOPS: 102.2012959757526, Tokens per sec: 83511.3098371969, Loss: 2.342721939086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3210 , TFLOPS: 102.8108880728407, Tokens per sec: 84009.42323202417, Loss: 2.3317911624908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3211 , TFLOPS: 103.46834532069275, Tokens per sec: 84546.64847369929, Loss: 2.310225486755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3212 , TFLOPS: 102.76093902299685, Tokens per sec: 83968.60857759384, Loss: 2.3371777534484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3213 , TFLOPS: 102.66132261954326, Tokens per sec: 83887.20944997754, Loss: 2.341061592102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3214 , TFLOPS: 102.28221995855506, Tokens per sec: 83577.43490671342, Loss: 2.3221657276153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3215 , TFLOPS: 102.48997204424087, Tokens per sec: 83747.19448394174, Loss: 2.340559482574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3216 , TFLOPS: 102.7078842614192, Tokens per sec: 83925.25616615766, Loss: 2.325443744659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3217 , TFLOPS: 101.27717012484246, Tokens per sec: 82756.18281530247, Loss: 2.3359568119049072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3218 , TFLOPS: 103.26648719532575, Tokens per sec: 84381.70500317149, Loss: 2.334242343902588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3219 , TFLOPS: 103.08248745050375, Tokens per sec: 84231.35407510262, Loss: 2.3478314876556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3220 , TFLOPS: 101.2893013997336, Tokens per sec: 82766.09559230281, Loss: 2.3311543464660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3221 , TFLOPS: 102.59525755823911, Tokens per sec: 83833.22598771995, Loss: 2.3410232067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3222 , TFLOPS: 101.42127010820434, Tokens per sec: 82873.93062117104, Loss: 2.325216770172119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3223 , TFLOPS: 102.8292072065397, Tokens per sec: 84024.3922677461, Loss: 2.3391165733337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3224 , TFLOPS: 102.81469982100157, Tokens per sec: 84012.53790956954, Loss: 2.336371898651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3225 , TFLOPS: 101.67867789528191, Tokens per sec: 83084.26515025804, Loss: 2.3367316722869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3226 , TFLOPS: 101.95359106786123, Tokens per sec: 83308.90378046724, Loss: 2.337782859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3227 , TFLOPS: 101.42186746885113, Tokens per sec: 82874.41873993295, Loss: 2.3347537517547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3228 , TFLOPS: 101.97615488854137, Tokens per sec: 83327.3412591894, Loss: 2.3287672996520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3229 , TFLOPS: 102.14385583239915, Tokens per sec: 83464.37401742196, Loss: 2.3654062747955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3230 , TFLOPS: 99.87553153778744, Tokens per sec: 81610.8678444327, Loss: 2.3520116806030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3231 , TFLOPS: 103.45035679501424, Tokens per sec: 84531.94958630152, Loss: 2.339428186416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3232 , TFLOPS: 102.09144903946469, Tokens per sec: 83421.55107784433, Loss: 2.340952157974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3233 , TFLOPS: 99.74154897374018, Tokens per sec: 81501.38724233164, Loss: 2.342160224914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3234 , TFLOPS: 103.3885546316426, Tokens per sec: 84481.44944768136, Loss: 2.341007709503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3235 , TFLOPS: 101.42568628692608, Tokens per sec: 82877.53919448712, Loss: 2.304896831512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3236 , TFLOPS: 101.42742015096816, Tokens per sec: 82878.95597942953, Loss: 2.3507330417633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3237 , TFLOPS: 102.82966152172445, Tokens per sec: 84024.76350037866, Loss: 2.3283684253692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3238 , TFLOPS: 100.91821229889311, Tokens per sec: 82462.86913532276, Loss: 2.3399577140808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3239 , TFLOPS: 102.15421431428581, Tokens per sec: 83472.83819962258, Loss: 2.3318562507629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3240 , TFLOPS: 100.69137134309386, Tokens per sec: 82277.5115509335, Loss: 2.341654062271118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3241 , TFLOPS: 101.00770924793036, Tokens per sec: 82535.99939623753, Loss: 2.3288707733154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3242 , TFLOPS: 100.82221405003492, Tokens per sec: 82384.42649496596, Loss: 2.343003988265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3243 , TFLOPS: 100.86224405515931, Tokens per sec: 82417.1360426171, Loss: 2.3482775688171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3244 , TFLOPS: 101.98677893276384, Tokens per sec: 83336.02243921082, Loss: 2.321516513824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3245 , TFLOPS: 98.65182066743405, Tokens per sec: 80610.9421911467, Loss: 2.3367412090301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3246 , TFLOPS: 101.48201425268172, Tokens per sec: 82923.5662253169, Loss: 2.332381010055542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3247 , TFLOPS: 101.5045903228593, Tokens per sec: 82942.01371341874, Loss: 2.314365863800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3248 , TFLOPS: 102.8917445634088, Tokens per sec: 84075.49315189861, Loss: 2.332859516143799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3249 , TFLOPS: 103.39036166193344, Tokens per sec: 84482.92601864909, Loss: 2.3251748085021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3250 , TFLOPS: 102.89153714548071, Tokens per sec: 84075.32366537016, Loss: 2.339399576187134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3251 , TFLOPS: 102.70066280793111, Tokens per sec: 83919.3553306158, Loss: 2.31929874420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3252 , TFLOPS: 102.73858493069136, Tokens per sec: 83950.3424733255, Loss: 2.3398172855377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3253 , TFLOPS: 103.36682195823748, Tokens per sec: 84463.69112079323, Loss: 2.3275437355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3254 , TFLOPS: 102.1081006183952, Tokens per sec: 83435.15751163823, Loss: 2.3399620056152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3255 , TFLOPS: 101.90607294545521, Tokens per sec: 83270.07549942398, Loss: 2.334050178527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3256 , TFLOPS: 102.6741077451533, Tokens per sec: 83897.6565052319, Loss: 2.316070318222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3257 , TFLOPS: 103.38032322739521, Tokens per sec: 84474.7233553756, Loss: 2.314229965209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3258 , TFLOPS: 101.40033330487702, Tokens per sec: 82856.82262021102, Loss: 2.353443145751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3259 , TFLOPS: 102.62848978024424, Tokens per sec: 83860.38089179383, Loss: 2.322735071182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3260 , TFLOPS: 102.26977169262425, Tokens per sec: 83567.26310817446, Loss: 2.3344929218292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3261 , TFLOPS: 102.70220359158965, Tokens per sec: 83920.6143446065, Loss: 2.357341766357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3262 , TFLOPS: 101.33241173039137, Tokens per sec: 82801.32215324191, Loss: 2.3542115688323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3263 , TFLOPS: 102.77354055142156, Tokens per sec: 83978.90561086187, Loss: 2.3305137157440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3264 , TFLOPS: 102.23600065516007, Tokens per sec: 83539.66792411865, Loss: 2.3180019855499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3265 , TFLOPS: 101.37154038466836, Tokens per sec: 82833.2951839128, Loss: 2.3114407062530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3266 , TFLOPS: 102.1166854462659, Tokens per sec: 83442.17239548451, Loss: 2.32576322555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3267 , TFLOPS: 101.94345167773648, Tokens per sec: 83300.6186237852, Loss: 2.341675281524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3268 , TFLOPS: 100.63930941911565, Tokens per sec: 82234.97041265776, Loss: 2.3319954872131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3269 , TFLOPS: 103.29348802538014, Tokens per sec: 84403.76807647213, Loss: 2.336967945098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3270 , TFLOPS: 102.33190581922365, Tokens per sec: 83618.03450249355, Loss: 2.3123581409454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3271 , TFLOPS: 99.65727761583089, Tokens per sec: 81432.5269464462, Loss: 2.317234992980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3272 , TFLOPS: 102.60463207533141, Tokens per sec: 83840.88614695761, Loss: 2.351475954055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3273 , TFLOPS: 101.37199199730077, Tokens per sec: 82833.6642082203, Loss: 2.33709979057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3274 , TFLOPS: 101.33869009382293, Tokens per sec: 82806.4523656213, Loss: 2.3219895362854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3275 , TFLOPS: 102.16949559413929, Tokens per sec: 83485.324926766, Loss: 2.3235464096069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3276 , TFLOPS: 102.39173949987604, Tokens per sec: 83666.92614321058, Loss: 2.330925703048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3277 , TFLOPS: 100.83542505755823, Tokens per sec: 82395.22154929483, Loss: 2.335857391357422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3278 , TFLOPS: 101.3260266044087, Tokens per sec: 82796.1047024337, Loss: 2.355961799621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3279 , TFLOPS: 100.40292978669311, Tokens per sec: 82041.81853005223, Loss: 2.3453428745269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3280 , TFLOPS: 102.30093271304584, Tokens per sec: 83592.72558011698, Loss: 2.336118221282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3281 , TFLOPS: 100.82639332710143, Tokens per sec: 82387.84148984116, Loss: 2.3496885299682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3282 , TFLOPS: 102.67799293776211, Tokens per sec: 83900.83119612634, Loss: 2.325251340866089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3283 , TFLOPS: 101.22564715403695, Tokens per sec: 82714.08206953802, Loss: 2.3400661945343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3284 , TFLOPS: 101.97043122520981, Tokens per sec: 83322.66430654099, Loss: 2.325488567352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3285 , TFLOPS: 101.5347460321759, Tokens per sec: 82966.65471977835, Loss: 2.3281521797180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3286 , TFLOPS: 102.8190268443221, Tokens per sec: 84016.07363171224, Loss: 2.330988645553589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3287 , TFLOPS: 101.80779012821463, Tokens per sec: 83189.76607943192, Loss: 2.3441996574401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3288 , TFLOPS: 102.19665066848967, Tokens per sec: 83507.5140370515, Loss: 2.338289737701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3289 , TFLOPS: 102.71193582547787, Tokens per sec: 83928.56680344655, Loss: 2.329101800918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3290 , TFLOPS: 102.72118618809857, Tokens per sec: 83936.12551286945, Loss: 2.3366427421569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3291 , TFLOPS: 103.30687008849753, Tokens per sec: 84414.70290473031, Loss: 2.338850259780884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3292 , TFLOPS: 101.99037036775772, Tokens per sec: 83338.95709319608, Loss: 2.318708658218384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3293 , TFLOPS: 102.78392350362131, Tokens per sec: 83987.38978838523, Loss: 2.336796522140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3294 , TFLOPS: 103.37460093049991, Tokens per sec: 84470.04751927742, Loss: 2.3346707820892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3295 , TFLOPS: 103.28413888276039, Tokens per sec: 84396.12865136919, Loss: 2.3551383018493652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3296 , TFLOPS: 101.35210436025253, Tokens per sec: 82817.41350803485, Loss: 2.3352839946746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3297 , TFLOPS: 102.17684852210235, Tokens per sec: 83491.33319348581, Loss: 2.3541762828826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3298 , TFLOPS: 102.01859926524878, Tokens per sec: 83362.02365201249, Loss: 2.316349506378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3299 , TFLOPS: 103.34075126231329, Tokens per sec: 84442.38808403416, Loss: 2.314756393432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3300 , TFLOPS: 101.61408419594838, Tokens per sec: 83031.48397574286, Loss: 2.314971685409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3301 , TFLOPS: 102.76126258036177, Tokens per sec: 83968.87296464581, Loss: 2.3291001319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3302 , TFLOPS: 102.73142632215186, Tokens per sec: 83944.49299000889, Loss: 2.3547966480255127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3303 , TFLOPS: 101.37247307429364, Tokens per sec: 82834.0573086152, Loss: 2.352501392364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3304 , TFLOPS: 101.39408807517115, Tokens per sec: 82851.71948225184, Loss: 2.310741901397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3305 , TFLOPS: 102.54768625278517, Tokens per sec: 83794.35424943919, Loss: 2.2995619773864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3306 , TFLOPS: 101.01992415347283, Tokens per sec: 82545.98050999592, Loss: 2.329373836517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3307 , TFLOPS: 103.36946961739143, Tokens per sec: 84465.85459124473, Loss: 2.312224864959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3308 , TFLOPS: 100.79247262692724, Tokens per sec: 82360.12401253234, Loss: 2.314417839050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3309 , TFLOPS: 101.24058816509141, Tokens per sec: 82726.29075428647, Loss: 2.3194308280944824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3310 , TFLOPS: 102.05302253727035, Tokens per sec: 83390.15179371517, Loss: 2.335278272628784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3311 , TFLOPS: 102.05962839822847, Tokens per sec: 83395.54961275404, Loss: 2.326672077178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3312 , TFLOPS: 101.47090953726004, Tokens per sec: 82914.49227648503, Loss: 2.327177047729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3313 , TFLOPS: 101.52147921745706, Tokens per sec: 82955.81406395338, Loss: 2.3045318126678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3314 , TFLOPS: 103.40010988288219, Tokens per sec: 84490.89154092784, Loss: 2.3287606239318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3315 , TFLOPS: 100.28367522710812, Tokens per sec: 81944.37255952986, Loss: 2.3426735401153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3316 , TFLOPS: 102.12927968100075, Tokens per sec: 83452.4634688908, Loss: 2.355837345123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3317 , TFLOPS: 100.78134624805752, Tokens per sec: 82351.0323619394, Loss: 2.309903144836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3318 , TFLOPS: 102.14583052763756, Tokens per sec: 83465.98759173442, Loss: 2.344069004058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3319 , TFLOPS: 102.71421761877671, Tokens per sec: 83930.43131548962, Loss: 2.3400468826293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3320 , TFLOPS: 102.68451837673693, Tokens per sec: 83906.1633002924, Loss: 2.3338699340820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3321 , TFLOPS: 102.12326806864777, Tokens per sec: 83447.55122568451, Loss: 2.329965829849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3322 , TFLOPS: 102.68197966875782, Tokens per sec: 83904.08885665052, Loss: 2.3305909633636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3323 , TFLOPS: 102.12155253148933, Tokens per sec: 83446.14941611048, Loss: 2.316697359085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3324 , TFLOPS: 102.16933381707713, Tokens per sec: 83485.19273456399, Loss: 2.325930118560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3325 , TFLOPS: 100.63677860737558, Tokens per sec: 82232.90242123605, Loss: 2.346560478210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3326 , TFLOPS: 102.81890898750773, Tokens per sec: 84015.9773278753, Loss: 2.3286452293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3327 , TFLOPS: 102.71808297943647, Tokens per sec: 83933.58980118793, Loss: 2.3289918899536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3328 , TFLOPS: 101.22929484647408, Tokens per sec: 82717.06269292835, Loss: 2.3272688388824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3329 , TFLOPS: 102.67784265222821, Tokens per sec: 83900.70839394789, Loss: 2.3590455055236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3330 , TFLOPS: 101.28760935198812, Tokens per sec: 82764.7129765328, Loss: 2.3304667472839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3331 , TFLOPS: 102.72290473221744, Tokens per sec: 83937.52977950823, Loss: 2.2982826232910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3332 , TFLOPS: 102.86050643053072, Tokens per sec: 84049.96766938284, Loss: 2.309328556060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3333 , TFLOPS: 102.81364037411034, Tokens per sec: 84011.67220824212, Loss: 2.338414430618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3334 , TFLOPS: 101.97394222469019, Tokens per sec: 83325.53323459758, Loss: 2.342883586883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3335 , TFLOPS: 100.93080507043071, Tokens per sec: 82473.1590131129, Loss: 2.3300724029541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3336 , TFLOPS: 103.43446937745017, Tokens per sec: 84518.96756843141, Loss: 2.3256094455718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3337 , TFLOPS: 102.1721241927922, Tokens per sec: 83487.47282239143, Loss: 2.356504440307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3338 , TFLOPS: 100.98284162684793, Tokens per sec: 82515.67942290155, Loss: 2.3588361740112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3339 , TFLOPS: 102.5428129315229, Tokens per sec: 83790.37213318534, Loss: 2.340531826019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3340 , TFLOPS: 101.64697961805378, Tokens per sec: 83058.36367194868, Loss: 2.331291913986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3341 , TFLOPS: 101.40624765919394, Tokens per sec: 82861.65539137239, Loss: 2.329145669937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3342 , TFLOPS: 102.09243305039874, Tokens per sec: 83422.35513850923, Loss: 2.3062121868133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3343 , TFLOPS: 102.1191031937974, Tokens per sec: 83444.14799923085, Loss: 2.3449699878692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3344 , TFLOPS: 101.54129965801467, Tokens per sec: 82972.00985615654, Loss: 2.338019609451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3345 , TFLOPS: 101.97515620747775, Tokens per sec: 83326.52521118398, Loss: 2.316915512084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3346 , TFLOPS: 102.24755487711167, Tokens per sec: 83549.10917630744, Loss: 2.3336172103881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3347 , TFLOPS: 101.6467557279079, Tokens per sec: 83058.18072554703, Loss: 2.3482697010040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3348 , TFLOPS: 101.614425355888, Tokens per sec: 83031.76274631177, Loss: 2.3379292488098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3349 , TFLOPS: 102.2216090179044, Tokens per sec: 83527.90814684334, Loss: 2.3321053981781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3350 , TFLOPS: 100.79199119917787, Tokens per sec: 82359.73062552528, Loss: 2.3270561695098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3351 , TFLOPS: 101.63036269893907, Tokens per sec: 83044.7855595825, Loss: 2.31730318069458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3352 , TFLOPS: 103.36008811239815, Tokens per sec: 84458.1887220133, Loss: 2.3345389366149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3353 , TFLOPS: 101.15131650455929, Tokens per sec: 82653.34458240868, Loss: 2.320193290710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3354 , TFLOPS: 101.15511666424572, Tokens per sec: 82656.44979071329, Loss: 2.32060170173645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3355 , TFLOPS: 101.60150540821138, Tokens per sec: 83021.20552447617, Loss: 2.325763463973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3356 , TFLOPS: 102.17147950527261, Tokens per sec: 83486.94603162346, Loss: 2.326353073120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3357 , TFLOPS: 101.91577770515741, Tokens per sec: 83278.0055083994, Loss: 2.338083267211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3358 , TFLOPS: 102.69925248191713, Tokens per sec: 83918.20291692475, Loss: 2.3370776176452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3359 , TFLOPS: 102.64068624158321, Tokens per sec: 83870.34693431876, Loss: 2.325263023376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3360 , TFLOPS: 102.09219965710395, Tokens per sec: 83422.16442683998, Loss: 2.3296213150024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3361 , TFLOPS: 102.21685635451526, Tokens per sec: 83524.02462324363, Loss: 2.3225557804107666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3362 , TFLOPS: 101.31526401919383, Tokens per sec: 82787.31031700104, Loss: 2.3331284523010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3363 , TFLOPS: 102.1547497501562, Tokens per sec: 83473.27571805562, Loss: 2.3362982273101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3364 , TFLOPS: 101.52124598808757, Tokens per sec: 82955.62348623172, Loss: 2.3337700366973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3365 , TFLOPS: 103.22230492632464, Tokens per sec: 84345.60253381789, Loss: 2.3524489402770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3366 , TFLOPS: 101.31906898898444, Tokens per sec: 82790.41945576559, Loss: 2.3208203315734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3367 , TFLOPS: 101.6042154491367, Tokens per sec: 83023.4199686789, Loss: 2.3504865169525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3368 , TFLOPS: 102.70866475204355, Tokens per sec: 83925.89392513844, Loss: 2.303349494934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3369 , TFLOPS: 102.1944064928882, Tokens per sec: 83505.68026339731, Loss: 2.33054780960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3370 , TFLOPS: 101.4879048530738, Tokens per sec: 82928.37958653481, Loss: 2.3307528495788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3371 , TFLOPS: 102.0940730914417, Tokens per sec: 83423.69525826385, Loss: 2.334331512451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3372 , TFLOPS: 101.498400929276, Tokens per sec: 82936.95620059266, Loss: 2.3193488121032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3373 , TFLOPS: 101.52065092258967, Tokens per sec: 82955.13724289504, Loss: 2.3385276794433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3374 , TFLOPS: 102.73321297877914, Tokens per sec: 83945.95291313167, Loss: 2.3465919494628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3375 , TFLOPS: 102.08183760651697, Tokens per sec: 83413.69732856244, Loss: 2.3406386375427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3376 , TFLOPS: 100.9244360964746, Tokens per sec: 82467.9547605414, Loss: 2.332535743713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3377 , TFLOPS: 102.78168548387467, Tokens per sec: 83985.56104483851, Loss: 2.333493709564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3378 , TFLOPS: 101.06595367824588, Tokens per sec: 82583.59241959336, Loss: 2.332138776779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3379 , TFLOPS: 102.16942236391854, Tokens per sec: 83485.26508846751, Loss: 2.32951021194458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3380 , TFLOPS: 101.04667732926748, Tokens per sec: 82567.84121862573, Loss: 2.3454811573028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3381 , TFLOPS: 102.56605170796776, Tokens per sec: 83809.36113563775, Loss: 2.321754217147827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3382 , TFLOPS: 101.05240260011911, Tokens per sec: 82572.51948482018, Loss: 2.3630104064941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3383 , TFLOPS: 102.18034255116753, Tokens per sec: 83494.18825457929, Loss: 2.3184030055999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3384 , TFLOPS: 103.24558860788109, Tokens per sec: 84364.62822939295, Loss: 2.3099613189697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3385 , TFLOPS: 101.22550157503927, Tokens per sec: 82713.96311319148, Loss: 2.328892707824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3386 , TFLOPS: 102.23835882554643, Tokens per sec: 83541.59484584605, Loss: 2.340343713760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3387 , TFLOPS: 102.03049815017047, Tokens per sec: 83371.74653718655, Loss: 2.305394172668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3388 , TFLOPS: 101.38335241361098, Tokens per sec: 82842.9470968305, Loss: 2.3243203163146973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3389 , TFLOPS: 102.76722550250918, Tokens per sec: 83973.74542183174, Loss: 2.3432981967926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3390 , TFLOPS: 103.36675606514875, Tokens per sec: 84463.6372778542, Loss: 2.319279909133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3391 , TFLOPS: 101.39767815888445, Tokens per sec: 82854.65303207094, Loss: 2.314701557159424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3392 , TFLOPS: 103.28987910942129, Tokens per sec: 84400.81913834039, Loss: 2.3178184032440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3393 , TFLOPS: 101.0456437997684, Tokens per sec: 82566.99669506661, Loss: 2.3207831382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3394 , TFLOPS: 102.79311589616746, Tokens per sec: 83994.90112896793, Loss: 2.3323845863342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3395 , TFLOPS: 100.59789156076842, Tokens per sec: 82201.1268144117, Loss: 2.3256306648254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3396 , TFLOPS: 102.18623435500238, Tokens per sec: 83499.00259916126, Loss: 2.332404851913452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3397 , TFLOPS: 102.77881001628623, Tokens per sec: 83983.21142625088, Loss: 2.3393914699554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3398 , TFLOPS: 102.03340697595036, Tokens per sec: 83374.12341361113, Loss: 2.3372960090637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3399 , TFLOPS: 101.40318239693264, Tokens per sec: 82859.15068667181, Loss: 2.3214163780212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3400 , TFLOPS: 102.23023912301261, Tokens per sec: 83534.96002788593, Loss: 2.3175289630889893 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/optimizer.pt +[rank0]:[2024-08-27 13:16:25,966] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00779860200418625, 'preprocessing_with_comm': 0.0014950899931136519, 'state_converting': 3.355838526011212, : 3.366754262999166}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400, took 15.93s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001400 + eval ppl=8.636327743530273, eval loss=2.155977487564087 +------------------------------------------------------------------ +iteration: 3401 , TFLOPS: 101.34274173092543, Tokens per sec: 82809.76306259616, Loss: 2.3437418937683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3402 , TFLOPS: 100.84860586752195, Tokens per sec: 82405.99192842045, Loss: 2.3371493816375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3403 , TFLOPS: 102.15229572627138, Tokens per sec: 83471.27047196713, Loss: 2.3304858207702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3404 , TFLOPS: 101.56940370820736, Tokens per sec: 82994.9743990316, Loss: 2.334444522857666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3405 , TFLOPS: 102.24635454531573, Tokens per sec: 83548.12835429767, Loss: 2.3383982181549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3406 , TFLOPS: 101.91994469508069, Tokens per sec: 83281.41046313358, Loss: 2.351501703262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3407 , TFLOPS: 103.33591724254954, Tokens per sec: 84438.43808204646, Loss: 2.330322742462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3408 , TFLOPS: 102.16024318831914, Tokens per sec: 83477.76454779225, Loss: 2.307582139968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3409 , TFLOPS: 102.16172600221611, Tokens per sec: 83478.97619319847, Loss: 2.311943531036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3410 , TFLOPS: 103.21707657823997, Tokens per sec: 84341.33031600826, Loss: 2.324648380279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3411 , TFLOPS: 101.0256680368135, Tokens per sec: 82550.67398493428, Loss: 2.2907166481018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3412 , TFLOPS: 102.6046544272495, Tokens per sec: 83840.90441128527, Loss: 2.3284332752227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3413 , TFLOPS: 101.9582834641848, Tokens per sec: 83312.73805829627, Loss: 2.3389506340026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3414 , TFLOPS: 102.78953534471158, Tokens per sec: 83991.97536820175, Loss: 2.3193600177764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3415 , TFLOPS: 103.3681872262382, Tokens per sec: 84464.80671642175, Loss: 2.32861590385437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3416 , TFLOPS: 101.94569004897662, Tokens per sec: 83302.44765454631, Loss: 2.3316705226898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3417 , TFLOPS: 101.51035422228698, Tokens per sec: 82946.72354401705, Loss: 2.3456778526306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3418 , TFLOPS: 102.58668184908908, Tokens per sec: 83826.21855501508, Loss: 2.3158938884735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3419 , TFLOPS: 102.02677114678497, Tokens per sec: 83368.70110677865, Loss: 2.3192696571350098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3420 , TFLOPS: 102.7559526893703, Tokens per sec: 83964.53411602792, Loss: 2.331115245819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3421 , TFLOPS: 101.56841599780354, Tokens per sec: 82994.16731543471, Loss: 2.363938331604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3422 , TFLOPS: 101.9587171736677, Tokens per sec: 83313.0924534794, Loss: 2.3405189514160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3423 , TFLOPS: 102.24273351662623, Tokens per sec: 83545.16951854192, Loss: 2.318715810775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3424 , TFLOPS: 102.17754360515552, Tokens per sec: 83491.9011637416, Loss: 2.319993734359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3425 , TFLOPS: 101.52443374297934, Tokens per sec: 82958.2282828139, Loss: 2.324754476547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3426 , TFLOPS: 101.41097302170061, Tokens per sec: 82865.51660671824, Loss: 2.3178255558013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3427 , TFLOPS: 101.65855391148567, Tokens per sec: 83067.8213250607, Loss: 2.3055710792541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3428 , TFLOPS: 102.02610120617287, Tokens per sec: 83368.1536810587, Loss: 2.3435189723968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3429 , TFLOPS: 101.46424104728203, Tokens per sec: 82909.04328166178, Loss: 2.319812536239624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3430 , TFLOPS: 99.96683400881564, Tokens per sec: 81685.47344384452, Loss: 2.343313455581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3431 , TFLOPS: 103.39039249970635, Tokens per sec: 84482.95121698716, Loss: 2.342750310897827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3432 , TFLOPS: 99.13276410129693, Tokens per sec: 81003.93345154147, Loss: 2.3407039642333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3433 , TFLOPS: 102.05110407086882, Tokens per sec: 83388.58416543275, Loss: 2.3156344890594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3434 , TFLOPS: 101.51610040707507, Tokens per sec: 82951.41889952736, Loss: 2.327179431915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3435 , TFLOPS: 100.35032213718061, Tokens per sec: 81998.83146539419, Loss: 2.3328640460968018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3436 , TFLOPS: 102.24583514221825, Tokens per sec: 83547.70393665611, Loss: 2.320403575897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3437 , TFLOPS: 100.38752391209252, Tokens per sec: 82029.22999333359, Loss: 2.3214681148529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3438 , TFLOPS: 101.53374486391897, Tokens per sec: 82965.83663942324, Loss: 2.3390204906463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3439 , TFLOPS: 100.94556313587366, Tokens per sec: 82485.21820829275, Loss: 2.3515727519989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3440 , TFLOPS: 101.5997216645135, Tokens per sec: 83019.74798158297, Loss: 2.328749656677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3441 , TFLOPS: 101.53588532646296, Tokens per sec: 82967.58566646835, Loss: 2.331097364425659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3442 , TFLOPS: 101.04108757608272, Tokens per sec: 82563.27368740542, Loss: 2.3289613723754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3443 , TFLOPS: 102.30413706140794, Tokens per sec: 83595.34393565092, Loss: 2.315194606781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3444 , TFLOPS: 101.73294021298081, Tokens per sec: 83128.60428688613, Loss: 2.3497447967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3445 , TFLOPS: 102.79135248433279, Tokens per sec: 83993.46019976356, Loss: 2.3532257080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3446 , TFLOPS: 102.0980337580436, Tokens per sec: 83426.93162090088, Loss: 2.3255391120910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3447 , TFLOPS: 102.31149417944732, Tokens per sec: 83601.35562618996, Loss: 2.3343403339385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3448 , TFLOPS: 103.28502910505917, Tokens per sec: 84396.85607492588, Loss: 2.335118055343628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3449 , TFLOPS: 100.89892170305602, Tokens per sec: 82447.10629288017, Loss: 2.3293817043304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3450 , TFLOPS: 103.38239066586405, Tokens per sec: 84476.41271256909, Loss: 2.3259177207946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3451 , TFLOPS: 102.72946336174802, Tokens per sec: 83942.88900453181, Loss: 2.3156023025512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3452 , TFLOPS: 102.74516691076217, Tokens per sec: 83955.72077867658, Loss: 2.3424315452575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3453 , TFLOPS: 101.85270565842652, Tokens per sec: 83226.46771539665, Loss: 2.3240768909454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3454 , TFLOPS: 101.94519522462186, Tokens per sec: 83302.04332082812, Loss: 2.322492837905884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3455 , TFLOPS: 101.98648147544945, Tokens per sec: 83335.77937918199, Loss: 2.311406373977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3456 , TFLOPS: 102.20624366462596, Tokens per sec: 83515.35272114014, Loss: 2.304436206817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3457 , TFLOPS: 102.67083959273918, Tokens per sec: 83894.98601376462, Loss: 2.334810972213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3458 , TFLOPS: 101.9812537983409, Tokens per sec: 83331.50770964421, Loss: 2.3422231674194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3459 , TFLOPS: 100.72345295133093, Tokens per sec: 82303.72625887807, Loss: 2.3482863903045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3460 , TFLOPS: 102.11217877774804, Tokens per sec: 83438.48988062658, Loss: 2.3391337394714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3461 , TFLOPS: 102.72807894223817, Tokens per sec: 83941.75775971219, Loss: 2.338634490966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3462 , TFLOPS: 101.56234686564585, Tokens per sec: 82989.2080713153, Loss: 2.3412866592407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3463 , TFLOPS: 101.4781854453502, Tokens per sec: 82920.43760828405, Loss: 2.3311989307403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3464 , TFLOPS: 100.71229591686743, Tokens per sec: 82294.60955881033, Loss: 2.330564260482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3465 , TFLOPS: 102.13094244783504, Tokens per sec: 83453.8221584745, Loss: 2.323028326034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3466 , TFLOPS: 102.0886201435948, Tokens per sec: 83419.23951420678, Loss: 2.332427978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3467 , TFLOPS: 101.07446509132161, Tokens per sec: 82590.54731432081, Loss: 2.3421707153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3468 , TFLOPS: 100.78230279385522, Tokens per sec: 82351.8139801344, Loss: 2.3241753578186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3469 , TFLOPS: 102.1999088808203, Tokens per sec: 83510.1764062207, Loss: 2.332094669342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3470 , TFLOPS: 100.31263969817722, Tokens per sec: 81968.04017445224, Loss: 2.33678936958313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3471 , TFLOPS: 101.40393350749933, Tokens per sec: 82859.76443845117, Loss: 2.316331148147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3472 , TFLOPS: 101.54821063802834, Tokens per sec: 82977.65699583016, Loss: 2.3415749073028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3473 , TFLOPS: 102.06499833031303, Tokens per sec: 83399.93752249469, Loss: 2.324619770050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3474 , TFLOPS: 100.59665126263168, Tokens per sec: 82200.11333487608, Loss: 2.3082380294799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3475 , TFLOPS: 101.66067669276829, Tokens per sec: 83069.55590429198, Loss: 2.302194118499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3476 , TFLOPS: 101.24461643525044, Tokens per sec: 82729.5823575303, Loss: 2.3096225261688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3477 , TFLOPS: 101.08245932942866, Tokens per sec: 82597.07961206806, Loss: 2.3179235458374023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3478 , TFLOPS: 101.5323744944597, Tokens per sec: 82964.71687526163, Loss: 2.340453863143921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3479 , TFLOPS: 102.08201321623626, Tokens per sec: 83413.84082378469, Loss: 2.3357808589935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3480 , TFLOPS: 101.3732104558421, Tokens per sec: 82834.65984206033, Loss: 2.327767848968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3481 , TFLOPS: 102.75209593270053, Tokens per sec: 83961.38266087131, Loss: 2.3411624431610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3482 , TFLOPS: 101.5863566700569, Tokens per sec: 83008.82710056692, Loss: 2.3362598419189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3483 , TFLOPS: 102.81521191802048, Tokens per sec: 84012.95635722624, Loss: 2.3234946727752686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3484 , TFLOPS: 100.95922851410646, Tokens per sec: 82496.3845406249, Loss: 2.331920623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3485 , TFLOPS: 101.57637825672028, Tokens per sec: 83000.67348216235, Loss: 2.3095757961273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3486 , TFLOPS: 103.38733340566183, Tokens per sec: 84480.4515524953, Loss: 2.330500602722168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3487 , TFLOPS: 100.88640582847012, Tokens per sec: 82436.87924956875, Loss: 2.338373899459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3488 , TFLOPS: 103.3046118365235, Tokens per sec: 84412.85762890974, Loss: 2.316507577896118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3489 , TFLOPS: 102.71139329089783, Tokens per sec: 83928.1234844751, Loss: 2.3050737380981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3490 , TFLOPS: 102.67424558437116, Tokens per sec: 83897.76913720513, Loss: 2.331193685531616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3491 , TFLOPS: 102.74530186383035, Tokens per sec: 83955.83105230259, Loss: 2.308828115463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3492 , TFLOPS: 101.77747085669394, Tokens per sec: 83164.99142218489, Loss: 2.3278748989105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3493 , TFLOPS: 102.12323479925033, Tokens per sec: 83447.52404040343, Loss: 2.311464786529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3494 , TFLOPS: 102.22662332303445, Tokens per sec: 83532.0054646448, Loss: 2.3516364097595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3495 , TFLOPS: 99.76900526713858, Tokens per sec: 81523.82248645533, Loss: 2.3120150566101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3496 , TFLOPS: 102.159128748242, Tokens per sec: 83476.85391011697, Loss: 2.3333733081817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3497 , TFLOPS: 101.62035237615756, Tokens per sec: 83036.6058671491, Loss: 2.3236780166625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3498 , TFLOPS: 102.13111467950615, Tokens per sec: 83453.96289340661, Loss: 2.3483903408050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3499 , TFLOPS: 102.82338377767111, Tokens per sec: 84019.63379410916, Loss: 2.336376667022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3500 , TFLOPS: 101.57825013269279, Tokens per sec: 83002.20304020587, Loss: 2.3170900344848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3501 , TFLOPS: 100.80185138403894, Tokens per sec: 82367.78763639899, Loss: 2.316474437713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3502 , TFLOPS: 101.54456513870677, Tokens per sec: 82974.67816449101, Loss: 2.313453197479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3503 , TFLOPS: 102.18597783590589, Tokens per sec: 83498.79299080413, Loss: 2.325371742248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3504 , TFLOPS: 101.98600217773195, Tokens per sec: 83335.3877326788, Loss: 2.317196846008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3505 , TFLOPS: 100.44520617334464, Tokens per sec: 82076.36365387621, Loss: 2.3254928588867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3506 , TFLOPS: 100.97350621201578, Tokens per sec: 82508.05121512726, Loss: 2.3163576126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3507 , TFLOPS: 102.23780169390977, Tokens per sec: 83541.13959924389, Loss: 2.319697141647339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3508 , TFLOPS: 100.34731714786773, Tokens per sec: 81996.37601127134, Loss: 2.3099727630615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3509 , TFLOPS: 101.50822975000733, Tokens per sec: 82944.98758302859, Loss: 2.3181874752044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3510 , TFLOPS: 101.48839528671044, Tokens per sec: 82928.78033248418, Loss: 2.3288838863372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3511 , TFLOPS: 101.96435390069617, Tokens per sec: 83317.69836823664, Loss: 2.348597288131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3512 , TFLOPS: 100.85376401198936, Tokens per sec: 82410.2067810472, Loss: 2.326707124710083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3513 , TFLOPS: 101.6148663314206, Tokens per sec: 83032.12307877118, Loss: 2.34092378616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3514 , TFLOPS: 102.24964215593066, Tokens per sec: 83550.81474556177, Loss: 2.328331470489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3515 , TFLOPS: 99.59475114384794, Tokens per sec: 81381.43495661436, Loss: 2.339442014694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3516 , TFLOPS: 102.1639751587513, Tokens per sec: 83480.81403690182, Loss: 2.343794345855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3517 , TFLOPS: 102.29246217407176, Tokens per sec: 83585.80408467019, Loss: 2.3362042903900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3518 , TFLOPS: 101.48042094112468, Tokens per sec: 82922.26428942813, Loss: 2.3152213096618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3519 , TFLOPS: 102.25835806376386, Tokens per sec: 83557.93673821971, Loss: 2.354095220565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3520 , TFLOPS: 101.49405905370293, Tokens per sec: 82933.40834229207, Loss: 2.3316802978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3521 , TFLOPS: 102.7085104741424, Tokens per sec: 83925.76786069394, Loss: 2.3287718296051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3522 , TFLOPS: 101.39607078848242, Tokens per sec: 82853.33960833805, Loss: 2.3264904022216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3523 , TFLOPS: 101.61898236259613, Tokens per sec: 83035.48639380078, Loss: 2.3165040016174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3524 , TFLOPS: 102.57848394109007, Tokens per sec: 83819.51983335614, Loss: 2.346742868423462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3525 , TFLOPS: 100.90548546668876, Tokens per sec: 82452.46971311078, Loss: 2.3346710205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3526 , TFLOPS: 102.20961171104769, Tokens per sec: 83518.10483857256, Loss: 2.3220725059509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3527 , TFLOPS: 102.73665168562131, Tokens per sec: 83948.7627690126, Loss: 2.323935031890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3528 , TFLOPS: 101.565465766949, Tokens per sec: 82991.75660585778, Loss: 2.3187477588653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3529 , TFLOPS: 102.68902955100512, Tokens per sec: 83909.84949691489, Loss: 2.327113389968872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3530 , TFLOPS: 101.94825100910771, Tokens per sec: 83304.5402809944, Loss: 2.339665651321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3531 , TFLOPS: 101.4951212484355, Tokens per sec: 82934.27628895138, Loss: 2.323490858078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3532 , TFLOPS: 102.72087591460429, Tokens per sec: 83935.87198041011, Loss: 2.3337795734405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3533 , TFLOPS: 102.11756313904858, Tokens per sec: 83442.88958085106, Loss: 2.3316287994384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3534 , TFLOPS: 102.11581015844811, Tokens per sec: 83441.45717527666, Loss: 2.312412738800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3535 , TFLOPS: 100.68167581182776, Tokens per sec: 82269.58908275078, Loss: 2.324066162109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3536 , TFLOPS: 101.9444992085027, Tokens per sec: 83301.47458814012, Loss: 2.3299875259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3537 , TFLOPS: 102.17782345769523, Tokens per sec: 83492.12983845583, Loss: 2.3309624195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3538 , TFLOPS: 102.08962568296845, Tokens per sec: 83420.06116631387, Loss: 2.311237335205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3539 , TFLOPS: 100.3701852141755, Tokens per sec: 82015.06209692781, Loss: 2.3132877349853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3540 , TFLOPS: 101.80097755993101, Tokens per sec: 83184.19935451627, Loss: 2.330002784729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3541 , TFLOPS: 101.5630388683438, Tokens per sec: 82989.77352453352, Loss: 2.3157098293304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3542 , TFLOPS: 102.0084449193276, Tokens per sec: 83353.72627456418, Loss: 2.3252429962158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3543 , TFLOPS: 102.17435718839434, Tokens per sec: 83489.29746058036, Loss: 2.3293683528900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3544 , TFLOPS: 99.91311773769107, Tokens per sec: 81641.5804959288, Loss: 2.3141210079193115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3545 , TFLOPS: 102.82198309019168, Tokens per sec: 84018.48925631281, Loss: 2.3294436931610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3546 , TFLOPS: 100.99674098795376, Tokens per sec: 82527.03694866251, Loss: 2.3221845626831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3547 , TFLOPS: 100.86442537004186, Tokens per sec: 82418.91845115967, Loss: 2.3369688987731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3548 , TFLOPS: 102.02981975794711, Tokens per sec: 83371.19220543747, Loss: 2.3215131759643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3549 , TFLOPS: 102.21617261876186, Tokens per sec: 83523.4659251586, Loss: 2.309868812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3550 , TFLOPS: 101.24514253161801, Tokens per sec: 82730.01224441516, Loss: 2.3404974937438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3551 , TFLOPS: 102.20939428823202, Tokens per sec: 83517.92717679292, Loss: 2.3259685039520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3552 , TFLOPS: 100.83203650788052, Tokens per sec: 82392.45267812416, Loss: 2.3351614475250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3553 , TFLOPS: 101.74876703678434, Tokens per sec: 83141.53679203469, Loss: 2.3351168632507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3554 , TFLOPS: 101.46042289656563, Tokens per sec: 82905.92337242348, Loss: 2.3241004943847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3555 , TFLOPS: 100.93056471318627, Tokens per sec: 82472.96261102114, Loss: 2.341295003890991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3556 , TFLOPS: 102.02721058692629, Tokens per sec: 83369.06018463032, Loss: 2.3257968425750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3557 , TFLOPS: 102.17235211321189, Tokens per sec: 83487.65906203355, Loss: 2.3147592544555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3558 , TFLOPS: 100.28275783646632, Tokens per sec: 81943.62293602072, Loss: 2.308987855911255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3559 , TFLOPS: 103.40512233720604, Tokens per sec: 84494.9873463879, Loss: 2.3521900177001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3560 , TFLOPS: 101.00741378265279, Tokens per sec: 82535.75796395326, Loss: 2.322216510772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3561 , TFLOPS: 101.76695327465715, Tokens per sec: 83156.39723515598, Loss: 2.3181867599487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3562 , TFLOPS: 102.69648727242837, Tokens per sec: 83915.94339306866, Loss: 2.3394412994384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3563 , TFLOPS: 100.78008436814487, Tokens per sec: 82350.00124737911, Loss: 2.347198963165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3564 , TFLOPS: 102.62360154793944, Tokens per sec: 83856.3865913433, Loss: 2.332878828048706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3565 , TFLOPS: 102.70832930570292, Tokens per sec: 83925.61982329836, Loss: 2.325002908706665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3566 , TFLOPS: 102.27549611105766, Tokens per sec: 83571.94067783606, Loss: 2.321892023086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3567 , TFLOPS: 102.57987601662624, Tokens per sec: 83820.65733411198, Loss: 2.314404010772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3568 , TFLOPS: 101.40170939147515, Tokens per sec: 82857.94705599397, Loss: 2.3214285373687744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3569 , TFLOPS: 101.62786911899323, Tokens per sec: 83042.7479912183, Loss: 2.3352530002593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3570 , TFLOPS: 102.82992219138914, Tokens per sec: 84024.9765002719, Loss: 2.310877561569214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3571 , TFLOPS: 100.9531569573938, Tokens per sec: 82491.42331533921, Loss: 2.324328899383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3572 , TFLOPS: 102.65096153990798, Tokens per sec: 83878.74314508964, Loss: 2.312243938446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3573 , TFLOPS: 101.55708327273953, Tokens per sec: 82984.90705405462, Loss: 2.3177311420440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3574 , TFLOPS: 102.10695942681262, Tokens per sec: 83434.22501461921, Loss: 2.35396146774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3575 , TFLOPS: 102.87468891447507, Tokens per sec: 84061.5565421034, Loss: 2.3210201263427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3576 , TFLOPS: 102.15306442931268, Tokens per sec: 83471.89859901037, Loss: 2.3317508697509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3577 , TFLOPS: 100.36841505517057, Tokens per sec: 82013.61565443693, Loss: 2.3133063316345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3578 , TFLOPS: 102.74807392406596, Tokens per sec: 83958.09617408029, Loss: 2.327472448348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3579 , TFLOPS: 102.1343298245063, Tokens per sec: 83456.59007115143, Loss: 2.3272452354431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3580 , TFLOPS: 102.0419392612, Tokens per sec: 83381.09536352918, Loss: 2.319827079772949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3581 , TFLOPS: 102.21682255585294, Tokens per sec: 83523.9970054866, Loss: 2.3388264179229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3582 , TFLOPS: 100.3608030175993, Tokens per sec: 82007.39566258616, Loss: 2.3475191593170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3583 , TFLOPS: 103.42007642390048, Tokens per sec: 84507.20671557837, Loss: 2.3348119258880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3584 , TFLOPS: 101.11431533740915, Tokens per sec: 82623.10997623553, Loss: 2.3360486030578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3585 , TFLOPS: 100.92453733676534, Tokens per sec: 82468.03748658912, Loss: 2.3191068172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3586 , TFLOPS: 101.40547710884914, Tokens per sec: 82861.02575484992, Loss: 2.3333139419555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3587 , TFLOPS: 102.12210763685374, Tokens per sec: 83446.60300699342, Loss: 2.3337974548339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3588 , TFLOPS: 102.04640544883983, Tokens per sec: 83384.74480041959, Loss: 2.336439609527588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3589 , TFLOPS: 101.60401854557797, Tokens per sec: 83023.25907371231, Loss: 2.351524829864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3590 , TFLOPS: 101.01229016254419, Tokens per sec: 82539.74257949172, Loss: 2.326129913330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3591 , TFLOPS: 101.1065287578379, Tokens per sec: 82616.74736162348, Loss: 2.314239025115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3592 , TFLOPS: 102.08048798667748, Tokens per sec: 83412.59451944931, Loss: 2.318512439727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3593 , TFLOPS: 101.02352803053141, Tokens per sec: 82548.92533071253, Loss: 2.3298847675323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3594 , TFLOPS: 101.51300430440487, Tokens per sec: 82948.88899433477, Loss: 2.3191003799438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3595 , TFLOPS: 102.79613254229668, Tokens per sec: 83997.3661081755, Loss: 2.332658052444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3596 , TFLOPS: 100.02392938981701, Tokens per sec: 81732.1275494266, Loss: 2.3447000980377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3597 , TFLOPS: 103.28686680926465, Tokens per sec: 84398.3577103389, Loss: 2.320572853088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3598 , TFLOPS: 100.88590763679714, Tokens per sec: 82436.47216432817, Loss: 2.299314260482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3599 , TFLOPS: 101.56991339572605, Tokens per sec: 82995.39087782364, Loss: 2.342040538787842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3600 , TFLOPS: 102.56918451325375, Tokens per sec: 83811.92103148252, Loss: 2.334012031555176 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/optimizer.pt +[rank0]:[2024-08-27 14:09:21,295] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00772698997752741, 'preprocessing_with_comm': 0.001673181977821514, 'state_converting': 3.363607484003296, : 3.3745927889831364}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600, took 15.88s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001600 + eval ppl=8.234820365905762, eval loss=2.1083714962005615 +------------------------------------------------------------------ +iteration: 3601 , TFLOPS: 101.1464125220553, Tokens per sec: 82649.33741206492, Loss: 2.343441963195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3602 , TFLOPS: 99.65496980027245, Tokens per sec: 81430.64117094496, Loss: 2.2999775409698486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3603 , TFLOPS: 101.87514775979773, Tokens per sec: 83244.80573413816, Loss: 2.316046953201294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3604 , TFLOPS: 101.15784464336225, Tokens per sec: 82658.67889267382, Loss: 2.3488826751708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3605 , TFLOPS: 100.51966771495647, Tokens per sec: 82137.20809633784, Loss: 2.3338418006896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3606 , TFLOPS: 101.29557970721127, Tokens per sec: 82771.22575896083, Loss: 2.336427927017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3607 , TFLOPS: 102.39250628003082, Tokens per sec: 83667.55269901373, Loss: 2.3231985569000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3608 , TFLOPS: 102.81934531582088, Tokens per sec: 84016.33386297207, Loss: 2.3356852531433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3609 , TFLOPS: 101.48564555043791, Tokens per sec: 82926.53345219139, Loss: 2.308837890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3610 , TFLOPS: 102.7722907767094, Tokens per sec: 83977.8843877725, Loss: 2.322517156600952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3611 , TFLOPS: 102.62824901608857, Tokens per sec: 83860.18415720442, Loss: 2.344082832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3612 , TFLOPS: 102.71191378792412, Tokens per sec: 83928.54879599408, Loss: 2.3401215076446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3613 , TFLOPS: 101.20357621595817, Tokens per sec: 82696.04733787735, Loss: 2.312793731689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3614 , TFLOPS: 101.15772287162841, Tokens per sec: 82658.57938985543, Loss: 2.328707695007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3615 , TFLOPS: 101.84543515418632, Tokens per sec: 83220.52679922257, Loss: 2.329836368560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3616 , TFLOPS: 102.73379653091227, Tokens per sec: 83946.42974860161, Loss: 2.3117377758026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3617 , TFLOPS: 100.70080038105245, Tokens per sec: 82285.2162605745, Loss: 2.3202409744262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3618 , TFLOPS: 102.77876023268009, Tokens per sec: 83983.17074678476, Loss: 2.319004774093628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3619 , TFLOPS: 100.2997446304166, Tokens per sec: 81957.50328263646, Loss: 2.305624008178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3620 , TFLOPS: 101.93858788506846, Tokens per sec: 83296.64429359099, Loss: 2.326906442642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3621 , TFLOPS: 102.77948591138714, Tokens per sec: 83983.76371753692, Loss: 2.3304250240325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3622 , TFLOPS: 102.11669647393461, Tokens per sec: 83442.18140647649, Loss: 2.3193576335906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3623 , TFLOPS: 101.06274336566332, Tokens per sec: 82580.96919054135, Loss: 2.3048999309539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3624 , TFLOPS: 101.69199249590447, Tokens per sec: 83095.14485317505, Loss: 2.332266330718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3625 , TFLOPS: 102.72437546797471, Tokens per sec: 83938.7315555556, Loss: 2.344202995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3626 , TFLOPS: 102.32481837036416, Tokens per sec: 83612.24316558329, Loss: 2.323878049850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3627 , TFLOPS: 100.54991641977087, Tokens per sec: 82161.92509171262, Loss: 2.295161485671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3628 , TFLOPS: 103.37544907858825, Tokens per sec: 84470.74056291368, Loss: 2.332418441772461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3629 , TFLOPS: 103.4242503787788, Tokens per sec: 84510.6173615566, Loss: 2.3340158462524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3630 , TFLOPS: 99.97544591954069, Tokens per sec: 81692.51045778826, Loss: 2.317464828491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3631 , TFLOPS: 102.64225407059386, Tokens per sec: 83871.62804776288, Loss: 2.3133702278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3632 , TFLOPS: 102.78038957493813, Tokens per sec: 83984.5021242868, Loss: 2.3287007808685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3633 , TFLOPS: 100.8420614779878, Tokens per sec: 82400.64433926452, Loss: 2.3308298587799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3634 , TFLOPS: 101.42578140438853, Tokens per sec: 82877.61691741423, Loss: 2.333285331726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3635 , TFLOPS: 102.27400779718604, Tokens per sec: 83570.72453825895, Loss: 2.3313345909118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3636 , TFLOPS: 100.5615179498416, Tokens per sec: 82171.40500058336, Loss: 2.3244972229003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3637 , TFLOPS: 102.12256626733426, Tokens per sec: 83446.97776576526, Loss: 2.3134865760803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3638 , TFLOPS: 101.51506075041509, Tokens per sec: 82950.56936930736, Loss: 2.3217875957489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3639 , TFLOPS: 101.44700442880624, Tokens per sec: 82894.9587969952, Loss: 2.3290951251983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3640 , TFLOPS: 98.80512559839698, Tokens per sec: 80736.21159665653, Loss: 2.350242853164673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3641 , TFLOPS: 102.87134085232097, Tokens per sec: 84058.82075433049, Loss: 2.3265719413757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3642 , TFLOPS: 101.13330807064078, Tokens per sec: 82638.62942747553, Loss: 2.326216697692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3643 , TFLOPS: 100.51302529713948, Tokens per sec: 82131.78040574852, Loss: 2.340846538543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3644 , TFLOPS: 101.48990774313232, Tokens per sec: 82930.01619956059, Loss: 2.303366184234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3645 , TFLOPS: 102.1951683755057, Tokens per sec: 83506.30281729667, Loss: 2.3062961101531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3646 , TFLOPS: 102.72635091205603, Tokens per sec: 83940.3457417669, Loss: 2.333979845046997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3647 , TFLOPS: 101.52258905178124, Tokens per sec: 82956.72093814934, Loss: 2.3435423374176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3648 , TFLOPS: 102.80887998999637, Tokens per sec: 84007.78237583941, Loss: 2.3470842838287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3649 , TFLOPS: 103.28164932515116, Tokens per sec: 84394.09436975978, Loss: 2.3148422241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3650 , TFLOPS: 102.00115413475473, Tokens per sec: 83347.76878680818, Loss: 2.3326027393341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3651 , TFLOPS: 102.05614659615225, Tokens per sec: 83392.70454264799, Loss: 2.3101401329040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3652 , TFLOPS: 101.38924230229557, Tokens per sec: 82847.7598764939, Loss: 2.301745891571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3653 , TFLOPS: 101.89149763249875, Tokens per sec: 83258.16563600535, Loss: 2.3131797313690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3654 , TFLOPS: 102.57146304798484, Tokens per sec: 83813.78288086601, Loss: 2.324286460876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3655 , TFLOPS: 100.66813425685172, Tokens per sec: 82258.52392958924, Loss: 2.324336051940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3656 , TFLOPS: 102.82082744623413, Tokens per sec: 84017.54494988619, Loss: 2.3215231895446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3657 , TFLOPS: 100.1934598636152, Tokens per sec: 81870.65526366976, Loss: 2.352240800857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3658 , TFLOPS: 102.63672722107853, Tokens per sec: 83867.11191675013, Loss: 2.314379930496216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3659 , TFLOPS: 101.34976305304681, Tokens per sec: 82815.50036564645, Loss: 2.314575433731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3660 , TFLOPS: 101.53230331127595, Tokens per sec: 82964.65870964977, Loss: 2.322305917739868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3661 , TFLOPS: 101.73170946933806, Tokens per sec: 83127.59861457348, Loss: 2.3229451179504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3662 , TFLOPS: 101.10214622909155, Tokens per sec: 82613.16628456849, Loss: 2.3487300872802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3663 , TFLOPS: 102.22174595064106, Tokens per sec: 83528.02003810742, Loss: 2.312958240509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3664 , TFLOPS: 102.93597015138967, Tokens per sec: 84111.63101830568, Loss: 2.326758623123169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3665 , TFLOPS: 101.04565228272308, Tokens per sec: 82567.00362670726, Loss: 2.317709445953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3666 , TFLOPS: 102.73115098113774, Tokens per sec: 83944.26800177834, Loss: 2.3212411403656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3667 , TFLOPS: 103.35432016097919, Tokens per sec: 84453.47558042858, Loss: 2.3284525871276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3668 , TFLOPS: 100.25482427395711, Tokens per sec: 81920.79770302058, Loss: 2.3134663105010986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3669 , TFLOPS: 103.3769943285521, Tokens per sec: 84472.00322643743, Loss: 2.329434633255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3670 , TFLOPS: 102.21875811253868, Tokens per sec: 83525.5785986802, Loss: 2.3265135288238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3671 , TFLOPS: 100.11897511179846, Tokens per sec: 81809.79185555219, Loss: 2.3412373065948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3672 , TFLOPS: 101.41470061759259, Tokens per sec: 82868.56252127844, Loss: 2.345045328140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3673 , TFLOPS: 102.1004993096971, Tokens per sec: 83428.94628662591, Loss: 2.3242061138153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3674 , TFLOPS: 101.51439728197633, Tokens per sec: 82950.02723216685, Loss: 2.320067882537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3675 , TFLOPS: 102.04779064368526, Tokens per sec: 83385.87667878586, Loss: 2.326030969619751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3676 , TFLOPS: 101.05364670166934, Tokens per sec: 82573.5360722229, Loss: 2.3463406562805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3677 , TFLOPS: 100.14109519635979, Tokens per sec: 81827.86674606889, Loss: 2.3311409950256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3678 , TFLOPS: 100.4223537511534, Tokens per sec: 82057.69035143006, Loss: 2.3205690383911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3679 , TFLOPS: 102.14714974114754, Tokens per sec: 83467.06555505302, Loss: 2.315246105194092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3680 , TFLOPS: 101.889926539495, Tokens per sec: 83256.88185546844, Loss: 2.3110976219177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3681 , TFLOPS: 100.49621285366796, Tokens per sec: 82118.04252539715, Loss: 2.311408519744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3682 , TFLOPS: 102.03186779323808, Tokens per sec: 83372.86570779484, Loss: 2.339282751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3683 , TFLOPS: 102.75758604323089, Tokens per sec: 83965.86877151369, Loss: 2.3267719745635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3684 , TFLOPS: 102.77758344859555, Tokens per sec: 83982.20916621615, Loss: 2.3065311908721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3685 , TFLOPS: 102.8294949867834, Tokens per sec: 84024.62742039133, Loss: 2.3281359672546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3686 , TFLOPS: 102.65964098897666, Tokens per sec: 83885.83534635256, Loss: 2.309995651245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3687 , TFLOPS: 102.73464357669597, Tokens per sec: 83947.12189151638, Loss: 2.338390350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3688 , TFLOPS: 102.84734966246242, Tokens per sec: 84039.2169355089, Loss: 2.2989602088928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3689 , TFLOPS: 101.28348113493041, Tokens per sec: 82761.33970410518, Loss: 2.343782424926758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3690 , TFLOPS: 101.30797283323307, Tokens per sec: 82781.35250126068, Loss: 2.3377187252044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3691 , TFLOPS: 102.66331420931856, Tokens per sec: 83888.83682925103, Loss: 2.3301503658294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3692 , TFLOPS: 102.00965833692794, Tokens per sec: 83354.71778932157, Loss: 2.3610079288482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3693 , TFLOPS: 101.51529649945391, Tokens per sec: 82950.76200591569, Loss: 2.3345940113067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3694 , TFLOPS: 102.04420049468197, Tokens per sec: 83382.94307561661, Loss: 2.3394875526428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3695 , TFLOPS: 100.92159480631078, Tokens per sec: 82465.63306920722, Loss: 2.3240530490875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3696 , TFLOPS: 101.9159974385849, Tokens per sec: 83278.18505823944, Loss: 2.303018808364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3697 , TFLOPS: 102.0045784507835, Tokens per sec: 83350.56688358502, Loss: 2.3277926445007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3698 , TFLOPS: 101.4557973107016, Tokens per sec: 82902.14368714081, Loss: 2.316938877105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3699 , TFLOPS: 101.67545283845037, Tokens per sec: 83081.62987330064, Loss: 2.310603380203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3700 , TFLOPS: 100.97559301212755, Tokens per sec: 82509.7563932176, Loss: 2.319002628326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3701 , TFLOPS: 102.27299645966798, Tokens per sec: 83569.89814833879, Loss: 2.3325624465942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3702 , TFLOPS: 102.12237763140142, Tokens per sec: 83446.82362648862, Loss: 2.3190345764160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3703 , TFLOPS: 100.5254554354849, Tokens per sec: 82141.93739177079, Loss: 2.3227503299713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3704 , TFLOPS: 102.72278692485989, Tokens per sec: 83937.43351608374, Loss: 2.309328556060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3705 , TFLOPS: 103.21394893580067, Tokens per sec: 84338.77463885843, Loss: 2.3476181030273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3706 , TFLOPS: 100.36481798507796, Tokens per sec: 82010.67639586743, Loss: 2.3189640045166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3707 , TFLOPS: 103.28145575838755, Tokens per sec: 84393.93620137438, Loss: 2.321362018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3708 , TFLOPS: 102.26745361116751, Tokens per sec: 83565.36894414344, Loss: 2.320822238922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3709 , TFLOPS: 100.94718220603056, Tokens per sec: 82486.54119219651, Loss: 2.3468282222747803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3710 , TFLOPS: 101.47979185428791, Tokens per sec: 82921.75024637968, Loss: 2.3001747131347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3711 , TFLOPS: 102.15963413418235, Tokens per sec: 83477.26687397862, Loss: 2.3608198165893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3712 , TFLOPS: 101.52602265948028, Tokens per sec: 82959.52662738916, Loss: 2.331580400466919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3713 , TFLOPS: 101.23931608030344, Tokens per sec: 82725.25130105983, Loss: 2.314091444015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3714 , TFLOPS: 101.57959503921388, Tokens per sec: 83003.30199794524, Loss: 2.313927173614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3715 , TFLOPS: 101.03691169848383, Tokens per sec: 82559.86147032272, Loss: 2.332263946533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3716 , TFLOPS: 100.40528530293304, Tokens per sec: 82043.74328300825, Loss: 2.330530881881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3717 , TFLOPS: 102.24608830295833, Tokens per sec: 83547.91080081377, Loss: 2.3445873260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3718 , TFLOPS: 101.70225695144136, Tokens per sec: 83103.53220402483, Loss: 2.330421209335327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3719 , TFLOPS: 99.67682211393506, Tokens per sec: 81448.49725896727, Loss: 2.324143886566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3720 , TFLOPS: 102.6596352588, Tokens per sec: 83885.83066407769, Loss: 2.3238563537597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3721 , TFLOPS: 102.89555284726859, Tokens per sec: 84078.60499867437, Loss: 2.3228986263275146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3722 , TFLOPS: 102.7981252396736, Tokens per sec: 83998.99439249877, Loss: 2.3228743076324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3723 , TFLOPS: 101.30077141873343, Tokens per sec: 82775.46804009214, Loss: 2.3411378860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3724 , TFLOPS: 102.7498344507703, Tokens per sec: 83959.53474577039, Loss: 2.320613145828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3725 , TFLOPS: 102.66250860430146, Tokens per sec: 83888.17854865344, Loss: 2.3161959648132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3726 , TFLOPS: 102.93280602539829, Tokens per sec: 84109.04552950627, Loss: 2.3289272785186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3727 , TFLOPS: 101.20347331128517, Tokens per sec: 82695.96325182005, Loss: 2.32258939743042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3728 , TFLOPS: 101.49504375941551, Tokens per sec: 82934.2129706784, Loss: 2.307720184326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3729 , TFLOPS: 102.9303853055028, Tokens per sec: 84107.0674969648, Loss: 2.3362815380096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3730 , TFLOPS: 101.49558699336718, Tokens per sec: 82934.65686112441, Loss: 2.3261373043060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3731 , TFLOPS: 101.98401792292346, Tokens per sec: 83333.76634699552, Loss: 2.323798656463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3732 , TFLOPS: 102.85943832341331, Tokens per sec: 84049.09489156169, Loss: 2.3141908645629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3733 , TFLOPS: 99.62482220498025, Tokens per sec: 81406.00679476355, Loss: 2.3033950328826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3734 , TFLOPS: 102.82044031041602, Tokens per sec: 84017.22861124344, Loss: 2.3414485454559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3735 , TFLOPS: 102.71868081675245, Tokens per sec: 83934.07830944854, Loss: 2.302459955215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3736 , TFLOPS: 100.6553823170137, Tokens per sec: 82248.10399128345, Loss: 2.3218188285827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3737 , TFLOPS: 101.70151158014811, Tokens per sec: 83102.92314195355, Loss: 2.3201777935028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3738 , TFLOPS: 100.15312374105088, Tokens per sec: 81837.69557957827, Loss: 2.334791421890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3739 , TFLOPS: 102.69852978833882, Tokens per sec: 83917.61238539814, Loss: 2.330979108810425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3740 , TFLOPS: 102.04637056939625, Tokens per sec: 83384.71629952836, Loss: 2.322946310043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3741 , TFLOPS: 100.31007660276977, Tokens per sec: 81965.94580321487, Loss: 2.3076391220092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3742 , TFLOPS: 100.00856802799072, Tokens per sec: 81719.57538524171, Loss: 2.3372488021850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3743 , TFLOPS: 102.19792175138963, Tokens per sec: 83508.55267160964, Loss: 2.316005229949951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3744 , TFLOPS: 100.38143688382944, Tokens per sec: 82024.25612584579, Loss: 2.3325583934783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3745 , TFLOPS: 102.61671134251458, Tokens per sec: 83850.75642712097, Loss: 2.2964160442352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3746 , TFLOPS: 102.2445063990146, Tokens per sec: 83546.61818637952, Loss: 2.330476760864258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3747 , TFLOPS: 100.57216003456655, Tokens per sec: 82180.10092196375, Loss: 2.3235292434692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3748 , TFLOPS: 101.62746560255277, Tokens per sec: 83042.41826754737, Loss: 2.326648235321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3749 , TFLOPS: 101.60616900597208, Tokens per sec: 83025.01627025811, Loss: 2.3233590126037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3750 , TFLOPS: 102.09143094162613, Tokens per sec: 83421.53628963456, Loss: 2.3285140991210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3751 , TFLOPS: 102.05922139405939, Tokens per sec: 83395.21703917027, Loss: 2.3325448036193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3752 , TFLOPS: 102.19772047140482, Tokens per sec: 83508.38820055264, Loss: 2.3162808418273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3753 , TFLOPS: 101.48520408576617, Tokens per sec: 82926.17272004379, Loss: 2.33205246925354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3754 , TFLOPS: 99.60271867538847, Tokens per sec: 81387.94543173864, Loss: 2.309213161468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3755 , TFLOPS: 102.73311152165586, Tokens per sec: 83945.8700099045, Loss: 2.311058759689331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3756 , TFLOPS: 101.8547498134093, Tokens per sec: 83228.13804705432, Loss: 2.3129706382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3757 , TFLOPS: 100.40725616857668, Tokens per sec: 82045.35372806028, Loss: 2.3076531887054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3758 , TFLOPS: 102.20235886301877, Tokens per sec: 83512.17834974322, Loss: 2.3164379596710205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3759 , TFLOPS: 102.78621634634794, Tokens per sec: 83989.2633292002, Loss: 2.325136184692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3760 , TFLOPS: 101.8735657172583, Tokens per sec: 83243.5130064538, Loss: 2.3137826919555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3761 , TFLOPS: 102.73632979500489, Tokens per sec: 83948.49974390371, Loss: 2.3099782466888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3762 , TFLOPS: 102.70920713283935, Tokens per sec: 83926.3371184488, Loss: 2.314420223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3763 , TFLOPS: 102.80039721997515, Tokens per sec: 84000.85088608915, Loss: 2.3023264408111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3764 , TFLOPS: 102.86154160508434, Tokens per sec: 84050.81353715838, Loss: 2.3175015449523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3765 , TFLOPS: 101.26413465294927, Tokens per sec: 82745.53119565563, Loss: 2.318932294845581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3766 , TFLOPS: 101.4190810418759, Tokens per sec: 82872.14187872248, Loss: 2.3323562145233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3767 , TFLOPS: 102.84361220898344, Tokens per sec: 84036.16296606061, Loss: 2.3024630546569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3768 , TFLOPS: 102.04435623354834, Tokens per sec: 83383.07033385341, Loss: 2.302969217300415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3769 , TFLOPS: 100.729982321193, Tokens per sec: 82309.06157507331, Loss: 2.3039865493774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3770 , TFLOPS: 102.77643650638875, Tokens per sec: 83981.27197021406, Loss: 2.337768793106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3771 , TFLOPS: 100.94950106570352, Tokens per sec: 82488.43599212806, Loss: 2.319302558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3772 , TFLOPS: 101.32989546131006, Tokens per sec: 82799.26604500104, Loss: 2.318260908126831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3773 , TFLOPS: 102.71302753499131, Tokens per sec: 83929.45886739308, Loss: 2.3246231079101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3774 , TFLOPS: 101.41129585186043, Tokens per sec: 82865.78039955221, Loss: 2.311225175857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3775 , TFLOPS: 100.94500441203812, Tokens per sec: 82484.76166066389, Loss: 2.334235668182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3776 , TFLOPS: 101.03876724751635, Tokens per sec: 82561.37768720328, Loss: 2.3091700077056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3777 , TFLOPS: 102.16963128853797, Tokens per sec: 83485.43580615225, Loss: 2.332975387573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3778 , TFLOPS: 102.1646216043435, Tokens per sec: 83481.34226423618, Loss: 2.311671495437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3779 , TFLOPS: 100.33846545396581, Tokens per sec: 81989.1430643212, Loss: 2.3333261013031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3780 , TFLOPS: 102.70100415589073, Tokens per sec: 83919.63425482067, Loss: 2.336699962615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3781 , TFLOPS: 102.02143133462052, Tokens per sec: 83364.33780880009, Loss: 2.3228156566619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3782 , TFLOPS: 100.24662156126173, Tokens per sec: 81914.09505532056, Loss: 2.316162347793579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3783 , TFLOPS: 102.710427522478, Tokens per sec: 83927.33433023817, Loss: 2.3139638900756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3784 , TFLOPS: 102.17458853230448, Tokens per sec: 83489.48649764467, Loss: 2.318540096282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3785 , TFLOPS: 99.73108367765974, Tokens per sec: 81492.83577950363, Loss: 2.3213274478912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3786 , TFLOPS: 101.57216585004134, Tokens per sec: 82997.23141622801, Loss: 2.32089900970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3787 , TFLOPS: 100.99484356820415, Tokens per sec: 82525.48651814117, Loss: 2.3264353275299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3788 , TFLOPS: 102.20845476205658, Tokens per sec: 83517.15946576942, Loss: 2.3412137031555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3789 , TFLOPS: 101.92757785554791, Tokens per sec: 83287.64771504636, Loss: 2.3336308002471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3790 , TFLOPS: 102.63923003960626, Tokens per sec: 83869.15703419743, Loss: 2.360123634338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3791 , TFLOPS: 101.37475803108968, Tokens per sec: 82835.92440563325, Loss: 2.326864242553711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3792 , TFLOPS: 100.92982447522424, Tokens per sec: 82472.35774352694, Loss: 2.348249912261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3793 , TFLOPS: 103.33915942828276, Tokens per sec: 84441.08735547037, Loss: 2.3084704875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3794 , TFLOPS: 101.75504658886678, Tokens per sec: 83146.66797569129, Loss: 2.3280696868896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3795 , TFLOPS: 100.10399209615007, Tokens per sec: 81797.54884776876, Loss: 2.3097410202026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3796 , TFLOPS: 101.25105540921047, Tokens per sec: 82734.84380890698, Loss: 2.31414532661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3797 , TFLOPS: 102.79768668275783, Tokens per sec: 83998.63603635345, Loss: 2.327233076095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3798 , TFLOPS: 102.80148393274919, Tokens per sec: 84001.73886707131, Loss: 2.318622589111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3799 , TFLOPS: 102.82637288373417, Tokens per sec: 84022.07626961972, Loss: 2.3225512504577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3800 , TFLOPS: 101.38707241594047, Tokens per sec: 82845.9868064953, Loss: 2.3178646564483643 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/optimizer.pt +[rank0]:[2024-08-27 15:02:17,583] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007986601995071396, 'preprocessing_with_comm': 0.0015467590128537267, 'state_converting': 3.342395305982791, : 3.3535918180132285}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800, took 15.80s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0001800 + eval ppl=8.26157283782959, eval loss=2.111614942550659 +------------------------------------------------------------------ +iteration: 3801 , TFLOPS: 100.59741072946888, Tokens per sec: 82200.73391477916, Loss: 2.3016366958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3802 , TFLOPS: 100.1809839061185, Tokens per sec: 81860.46083763948, Loss: 2.3358287811279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3803 , TFLOPS: 101.49676544444905, Tokens per sec: 82935.61980383944, Loss: 2.311366081237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3804 , TFLOPS: 100.88776267670589, Tokens per sec: 82437.98796519058, Loss: 2.3390276432037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3805 , TFLOPS: 102.28850348801141, Tokens per sec: 83582.56934038464, Loss: 2.2978501319885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3806 , TFLOPS: 101.70982075639053, Tokens per sec: 83109.71278375846, Loss: 2.3111348152160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3807 , TFLOPS: 102.11088153978048, Tokens per sec: 83437.42987408943, Loss: 2.3404462337493896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3808 , TFLOPS: 102.20716682656659, Tokens per sec: 83516.10706052613, Loss: 2.3374147415161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3809 , TFLOPS: 103.34438577752881, Tokens per sec: 84445.3579399775, Loss: 2.3040220737457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3810 , TFLOPS: 102.03683906695807, Tokens per sec: 83376.92786352341, Loss: 2.3252155780792236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3811 , TFLOPS: 103.34833744939708, Tokens per sec: 84448.58695278615, Loss: 2.3169240951538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3812 , TFLOPS: 101.99025184401309, Tokens per sec: 83338.86024439326, Loss: 2.3376424312591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3813 , TFLOPS: 103.32690826668599, Tokens per sec: 84431.07661595668, Loss: 2.3343496322631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3814 , TFLOPS: 102.62142382151053, Tokens per sec: 83854.60711501857, Loss: 2.3278164863586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3815 , TFLOPS: 102.7624085262237, Tokens per sec: 83969.80934650886, Loss: 2.323773145675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3816 , TFLOPS: 101.25472964441634, Tokens per sec: 82737.84612107693, Loss: 2.3206326961517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3817 , TFLOPS: 101.99261367650642, Tokens per sec: 83340.79015852255, Loss: 2.3097033500671387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3818 , TFLOPS: 102.04189180382733, Tokens per sec: 83381.0565848883, Loss: 2.340258836746216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3819 , TFLOPS: 103.34393827042231, Tokens per sec: 84444.99227040094, Loss: 2.357067823410034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3820 , TFLOPS: 102.04713439721776, Tokens per sec: 83385.340442904, Loss: 2.314256191253662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3821 , TFLOPS: 103.26121200800259, Tokens per sec: 84377.39451181433, Loss: 2.3137288093566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3822 , TFLOPS: 101.15801985433994, Tokens per sec: 82658.82206207402, Loss: 2.313154935836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3823 , TFLOPS: 101.35901526789303, Tokens per sec: 82823.06058857049, Loss: 2.3155453205108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3824 , TFLOPS: 102.77316111310323, Tokens per sec: 83978.59556204434, Loss: 2.3269026279449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3825 , TFLOPS: 101.48077176120604, Tokens per sec: 82922.55095354753, Loss: 2.316828489303589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3826 , TFLOPS: 101.63545476983576, Tokens per sec: 83048.94642179375, Loss: 2.293132781982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3827 , TFLOPS: 101.22791032030662, Tokens per sec: 82715.93136095605, Loss: 2.3255531787872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3828 , TFLOPS: 101.61037545656666, Tokens per sec: 83028.45346932262, Loss: 2.324871063232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3829 , TFLOPS: 102.09405189607317, Tokens per sec: 83423.6779389826, Loss: 2.3505399227142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3830 , TFLOPS: 101.56671956535492, Tokens per sec: 82992.78111681089, Loss: 2.3163483142852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3831 , TFLOPS: 100.14235061243097, Tokens per sec: 81828.89257885794, Loss: 2.328045129776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3832 , TFLOPS: 103.27400008216206, Tokens per sec: 84387.84397640434, Loss: 2.2978224754333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3833 , TFLOPS: 99.21194085929098, Tokens per sec: 81068.63081868947, Loss: 2.306316375732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3834 , TFLOPS: 101.96946272346091, Tokens per sec: 83321.8729188305, Loss: 2.325654983520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3835 , TFLOPS: 100.37140768631873, Tokens per sec: 82016.06101038527, Loss: 2.3166863918304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3836 , TFLOPS: 101.21352225916281, Tokens per sec: 82704.1745058138, Loss: 2.327331066131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3837 , TFLOPS: 101.00052073588331, Tokens per sec: 82530.12547799511, Loss: 2.3177738189697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3838 , TFLOPS: 100.6508861222834, Tokens per sec: 82244.43003482699, Loss: 2.319720506668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3839 , TFLOPS: 101.62267008094229, Tokens per sec: 83038.49972339172, Loss: 2.3113489151000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3840 , TFLOPS: 100.34618042024013, Tokens per sec: 81995.44716186484, Loss: 2.302219867706299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3841 , TFLOPS: 102.15717235542562, Tokens per sec: 83475.25529118467, Loss: 2.3150622844696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3842 , TFLOPS: 101.58634307637642, Tokens per sec: 83008.81599282067, Loss: 2.326361656188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3843 , TFLOPS: 102.22627271594794, Tokens per sec: 83531.71897456898, Loss: 2.3058481216430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3844 , TFLOPS: 102.17069263969724, Tokens per sec: 83486.30306350609, Loss: 2.3306403160095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3845 , TFLOPS: 102.90390369189407, Tokens per sec: 84085.42868878772, Loss: 2.3250129222869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3846 , TFLOPS: 100.92511483401908, Tokens per sec: 82468.50937446125, Loss: 2.308251142501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3847 , TFLOPS: 102.7945013783291, Tokens per sec: 83996.0332421077, Loss: 2.3147735595703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3848 , TFLOPS: 102.14529690088145, Tokens per sec: 83465.55155157523, Loss: 2.3284003734588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3849 , TFLOPS: 102.68883369412485, Tokens per sec: 83909.68945721623, Loss: 2.300194025039673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3850 , TFLOPS: 101.27388701207475, Tokens per sec: 82753.50009934512, Loss: 2.315131664276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3851 , TFLOPS: 103.44325814848128, Tokens per sec: 84526.14909948403, Loss: 2.3205337524414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3852 , TFLOPS: 103.37787483016417, Tokens per sec: 84472.72270697077, Loss: 2.3140721321105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3853 , TFLOPS: 102.67214416457708, Tokens per sec: 83896.05201299602, Loss: 2.3209850788116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3854 , TFLOPS: 101.48551198099207, Tokens per sec: 82926.42430915881, Loss: 2.3134210109710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3855 , TFLOPS: 102.04318423501122, Tokens per sec: 83382.11266367881, Loss: 2.3232944011688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3856 , TFLOPS: 102.69253509864669, Tokens per sec: 83912.71397013364, Loss: 2.3278422355651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3857 , TFLOPS: 102.67580785989023, Tokens per sec: 83899.04571274832, Loss: 2.3074638843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3858 , TFLOPS: 101.28176396935507, Tokens per sec: 82759.9365639098, Loss: 2.309846878051758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3859 , TFLOPS: 102.8106012789274, Tokens per sec: 84009.18888533486, Loss: 2.316943883895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3860 , TFLOPS: 101.67700865789247, Tokens per sec: 83082.90117341714, Loss: 2.312429904937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3861 , TFLOPS: 101.54661756544974, Tokens per sec: 82976.35525521684, Loss: 2.321007490158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3862 , TFLOPS: 102.79341204921128, Tokens per sec: 83995.1431232437, Loss: 2.295334577560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3863 , TFLOPS: 101.52323973746273, Tokens per sec: 82957.25263016984, Loss: 2.3019697666168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3864 , TFLOPS: 100.86774506942929, Tokens per sec: 82421.6310629858, Loss: 2.3157379627227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3865 , TFLOPS: 102.12461677389591, Tokens per sec: 83448.65328746152, Loss: 2.3287124633789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3866 , TFLOPS: 101.57795000916363, Tokens per sec: 83001.95780154433, Loss: 2.323273181915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3867 , TFLOPS: 102.16008838258556, Tokens per sec: 83477.63805204227, Loss: 2.3082008361816406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3868 , TFLOPS: 101.19856780400373, Tokens per sec: 82691.95483554139, Loss: 2.3233017921447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3869 , TFLOPS: 100.71564914397857, Tokens per sec: 82297.34956700259, Loss: 2.3434243202209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3870 , TFLOPS: 102.83749153674763, Tokens per sec: 84031.1616072166, Loss: 2.319516897201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3871 , TFLOPS: 99.78279929526185, Tokens per sec: 81535.09394192474, Loss: 2.3229787349700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3872 , TFLOPS: 102.77919068069546, Tokens per sec: 83983.52247693884, Loss: 2.3225808143615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3873 , TFLOPS: 100.34123694457999, Tokens per sec: 81991.40772064656, Loss: 2.317176580429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3874 , TFLOPS: 101.44121890672604, Tokens per sec: 82890.23129797088, Loss: 2.31772518157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3875 , TFLOPS: 101.63097171134943, Tokens per sec: 83045.2831993004, Loss: 2.315676689147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3876 , TFLOPS: 100.33315023156011, Tokens per sec: 81984.79985926751, Loss: 2.3118844032287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3877 , TFLOPS: 101.56987796722687, Tokens per sec: 82995.36192828495, Loss: 2.3444361686706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3878 , TFLOPS: 100.56580247359851, Tokens per sec: 82174.90599523859, Loss: 2.3384387493133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3879 , TFLOPS: 101.54582013621071, Tokens per sec: 82975.70365525798, Loss: 2.334188938140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3880 , TFLOPS: 101.87068501291745, Tokens per sec: 83241.15910878096, Loss: 2.2926063537597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3881 , TFLOPS: 101.53857571846648, Tokens per sec: 82969.78405503127, Loss: 2.3313331604003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3882 , TFLOPS: 102.97202456337482, Tokens per sec: 84141.09200646187, Loss: 2.330312490463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3883 , TFLOPS: 102.34089527844746, Tokens per sec: 83625.38002103461, Loss: 2.3228049278259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3884 , TFLOPS: 101.66438501767024, Tokens per sec: 83072.58607203014, Loss: 2.338944673538208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3885 , TFLOPS: 101.80252387488682, Tokens per sec: 83185.46288827235, Loss: 2.3401694297790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3886 , TFLOPS: 102.68549177507002, Tokens per sec: 83906.95868912789, Loss: 2.317842483520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3887 , TFLOPS: 102.67614113723444, Tokens per sec: 83899.31804224558, Loss: 2.339263916015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3888 , TFLOPS: 101.37160266420743, Tokens per sec: 82833.34607412742, Loss: 2.3026793003082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3889 , TFLOPS: 102.249058627858, Tokens per sec: 83550.33792975225, Loss: 2.343353271484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3890 , TFLOPS: 102.66866352451741, Tokens per sec: 83893.20789240363, Loss: 2.330578088760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3891 , TFLOPS: 103.31752261167216, Tokens per sec: 84423.40735563663, Loss: 2.3310322761535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3892 , TFLOPS: 101.54996931058398, Tokens per sec: 82979.09405244763, Loss: 2.309220314025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3893 , TFLOPS: 101.423352265123, Tokens per sec: 82875.63200518876, Loss: 2.3081517219543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3894 , TFLOPS: 102.66122610923865, Tokens per sec: 83887.13058892326, Loss: 2.3141086101531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3895 , TFLOPS: 102.73552104520492, Tokens per sec: 83947.83889362309, Loss: 2.2911787033081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3896 , TFLOPS: 101.70901234719238, Tokens per sec: 83109.05221179235, Loss: 2.3173277378082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3897 , TFLOPS: 102.798163938908, Tokens per sec: 83999.02601463941, Loss: 2.3187665939331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3898 , TFLOPS: 101.57144232226145, Tokens per sec: 82996.6402030538, Loss: 2.3445310592651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3899 , TFLOPS: 100.80807221060067, Tokens per sec: 82372.87083392091, Loss: 2.332287311553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3900 , TFLOPS: 102.31001940111139, Tokens per sec: 83600.15054684751, Loss: 2.3037900924682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3901 , TFLOPS: 101.40966017558233, Tokens per sec: 82864.44384635933, Loss: 2.3323023319244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3902 , TFLOPS: 101.66334489637529, Tokens per sec: 83071.73616214494, Loss: 2.324674606323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3903 , TFLOPS: 102.19912461514255, Tokens per sec: 83509.53556254662, Loss: 2.2952871322631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3904 , TFLOPS: 101.58893515929357, Tokens per sec: 83010.93405049843, Loss: 2.3178675174713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3905 , TFLOPS: 101.6054491859169, Tokens per sec: 83024.42808676121, Loss: 2.3415229320526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3906 , TFLOPS: 100.35984889216692, Tokens per sec: 82006.616022134, Loss: 2.32914662361145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3907 , TFLOPS: 101.09610630610005, Tokens per sec: 82608.2309080107, Loss: 2.3261046409606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3908 , TFLOPS: 102.81921797040701, Tokens per sec: 84016.22980575624, Loss: 2.316746950149536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3909 , TFLOPS: 99.88620267855345, Tokens per sec: 81619.5875082522, Loss: 2.3229870796203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3910 , TFLOPS: 102.64785461272902, Tokens per sec: 83876.20439491229, Loss: 2.33091402053833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3911 , TFLOPS: 100.94225747099662, Tokens per sec: 82482.51706442634, Loss: 2.325316905975342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3912 , TFLOPS: 101.59976925803086, Tokens per sec: 83019.78687147118, Loss: 2.3273348808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3913 , TFLOPS: 102.3412036650598, Tokens per sec: 83625.63201167411, Loss: 2.299687147140503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3914 , TFLOPS: 100.5728781775073, Tokens per sec: 82180.68773504728, Loss: 2.3165738582611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3915 , TFLOPS: 102.23564916368451, Tokens per sec: 83539.38071138578, Loss: 2.309372663497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3916 , TFLOPS: 100.51259231654969, Tokens per sec: 82131.42660616268, Loss: 2.329472064971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3917 , TFLOPS: 101.36423073023015, Tokens per sec: 82827.3222771041, Loss: 2.290886163711548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3918 , TFLOPS: 102.0673938028156, Tokens per sec: 83401.89492474147, Loss: 2.322976589202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3919 , TFLOPS: 102.8035972326585, Tokens per sec: 84003.46569882837, Loss: 2.312669515609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3920 , TFLOPS: 102.59664944657061, Tokens per sec: 83834.36333550602, Loss: 2.3111071586608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3921 , TFLOPS: 101.62116683070398, Tokens per sec: 83037.27137892491, Loss: 2.3284130096435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3922 , TFLOPS: 102.85696346483256, Tokens per sec: 84047.07262090682, Loss: 2.3420190811157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3923 , TFLOPS: 102.66072442565934, Tokens per sec: 83886.72065035612, Loss: 2.3312058448791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3924 , TFLOPS: 101.11615990588895, Tokens per sec: 82624.61722062378, Loss: 2.325878620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3925 , TFLOPS: 102.16775071155007, Tokens per sec: 83483.89913828507, Loss: 2.307516098022461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3926 , TFLOPS: 102.0332848833387, Tokens per sec: 83374.02364859519, Loss: 2.299433469772339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3927 , TFLOPS: 101.52952859643669, Tokens per sec: 82962.39141872709, Loss: 2.3188791275024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3928 , TFLOPS: 102.73464930185747, Tokens per sec: 83947.12656969321, Loss: 2.312173843383789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3929 , TFLOPS: 102.04544008873495, Tokens per sec: 83383.9559798273, Loss: 2.309117317199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3930 , TFLOPS: 100.84823707640227, Tokens per sec: 82405.69057970302, Loss: 2.3279151916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3931 , TFLOPS: 102.05295854858174, Tokens per sec: 83390.09950691048, Loss: 2.3088459968566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3932 , TFLOPS: 100.93594471702988, Tokens per sec: 82477.35875065537, Loss: 2.307802200317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3933 , TFLOPS: 103.36140457638241, Tokens per sec: 84459.26443862364, Loss: 2.320063591003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3934 , TFLOPS: 100.92679916339632, Tokens per sec: 82469.88568335389, Loss: 2.328028917312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3935 , TFLOPS: 102.74439030318484, Tokens per sec: 83955.08619263349, Loss: 2.299978017807007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3936 , TFLOPS: 101.67057020790654, Tokens per sec: 83077.64015019318, Loss: 2.302643299102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3937 , TFLOPS: 101.46910161010254, Tokens per sec: 82913.01497266444, Loss: 2.333253860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3938 , TFLOPS: 102.2703876563209, Tokens per sec: 83567.76642796733, Loss: 2.311683416366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3939 , TFLOPS: 101.26000264025986, Tokens per sec: 82742.15482171965, Loss: 2.333059549331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3940 , TFLOPS: 101.61843291912803, Tokens per sec: 83035.03742939905, Loss: 2.320847511291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3941 , TFLOPS: 101.48627927067673, Tokens per sec: 82927.05128131194, Loss: 2.3003861904144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3942 , TFLOPS: 102.21593992823371, Tokens per sec: 83523.27578773806, Loss: 2.325410842895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3943 , TFLOPS: 102.24394790466054, Tokens per sec: 83546.16182626589, Loss: 2.3219451904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3944 , TFLOPS: 102.19747342410234, Tokens per sec: 83508.18633184223, Loss: 2.291515350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3945 , TFLOPS: 101.48567646845447, Tokens per sec: 82926.55871609862, Loss: 2.3225502967834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3946 , TFLOPS: 103.29566539519993, Tokens per sec: 84405.54726140236, Loss: 2.3149147033691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3947 , TFLOPS: 99.59218548100424, Tokens per sec: 81379.33848745874, Loss: 2.3078107833862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3948 , TFLOPS: 103.36203063912001, Tokens per sec: 84459.77601060283, Loss: 2.329056978225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3949 , TFLOPS: 99.65544497970808, Tokens per sec: 81431.02945229401, Loss: 2.320366144180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3950 , TFLOPS: 101.45648586848992, Tokens per sec: 82902.7063254347, Loss: 2.3249990940093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3951 , TFLOPS: 102.17850039264674, Tokens per sec: 83492.68297943057, Loss: 2.339200496673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3952 , TFLOPS: 100.39856751058277, Tokens per sec: 82038.25400194748, Loss: 2.330383062362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3953 , TFLOPS: 102.22554130400236, Tokens per sec: 83531.12131904003, Loss: 2.334806442260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3954 , TFLOPS: 100.47279854801799, Tokens per sec: 82098.91009351272, Loss: 2.318406581878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3955 , TFLOPS: 102.0454792091888, Tokens per sec: 83383.9879461572, Loss: 2.3086462020874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3956 , TFLOPS: 101.44173852425291, Tokens per sec: 82890.65589082822, Loss: 2.3136608600616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3957 , TFLOPS: 102.80472182836937, Tokens per sec: 84004.38463493339, Loss: 2.3227481842041016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3958 , TFLOPS: 103.32932746520045, Tokens per sec: 84433.05340533856, Loss: 2.315542459487915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3959 , TFLOPS: 100.97127597042783, Tokens per sec: 82506.2288273142, Loss: 2.314210891723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3960 , TFLOPS: 102.87087808199905, Tokens per sec: 84058.44261278768, Loss: 2.3049702644348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3961 , TFLOPS: 103.42142852375527, Tokens per sec: 84508.31155117587, Loss: 2.3021693229675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3962 , TFLOPS: 100.73885386195978, Tokens per sec: 82316.31073940762, Loss: 2.318495035171509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3963 , TFLOPS: 102.93675875841555, Tokens per sec: 84112.27540940707, Loss: 2.316060781478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3964 , TFLOPS: 101.43061504938835, Tokens per sec: 82881.56661317268, Loss: 2.3113441467285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3965 , TFLOPS: 102.68280711862391, Tokens per sec: 83904.76498723656, Loss: 2.320050001144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3966 , TFLOPS: 101.89630368075959, Tokens per sec: 83262.09278175795, Loss: 2.319103240966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3967 , TFLOPS: 102.62219176998282, Tokens per sec: 83855.23462548404, Loss: 2.303114891052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3968 , TFLOPS: 100.17591296737562, Tokens per sec: 81856.31724305509, Loss: 2.301300525665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3969 , TFLOPS: 102.67081689689624, Tokens per sec: 83894.9674684071, Loss: 2.334223985671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3970 , TFLOPS: 100.30042858182543, Tokens per sec: 81958.0621569391, Loss: 2.3126626014709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3971 , TFLOPS: 103.40850394284318, Tokens per sec: 84497.7505434043, Loss: 2.314819574356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3972 , TFLOPS: 100.29582694422794, Tokens per sec: 81954.30204040151, Loss: 2.340932607650757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3973 , TFLOPS: 101.39105613702279, Tokens per sec: 82849.24200754178, Loss: 2.3130910396575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3974 , TFLOPS: 101.61873416297058, Tokens per sec: 83035.28358349751, Loss: 2.3025670051574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3975 , TFLOPS: 102.1362302671745, Tokens per sec: 83458.14297177736, Loss: 2.302722215652466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3976 , TFLOPS: 101.74517871562549, Tokens per sec: 83138.60468244419, Loss: 2.31451153755188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3977 , TFLOPS: 101.21008242532152, Tokens per sec: 82701.36372903288, Loss: 2.327598810195923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3978 , TFLOPS: 101.38181677038212, Tokens per sec: 82841.69228321723, Loss: 2.305124044418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3979 , TFLOPS: 102.17373278434633, Tokens per sec: 83488.78724395916, Loss: 2.3168392181396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3980 , TFLOPS: 102.23136731319752, Tokens per sec: 83535.88190112822, Loss: 2.3033840656280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3981 , TFLOPS: 102.15262467128298, Tokens per sec: 83471.53926140392, Loss: 2.3329992294311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3982 , TFLOPS: 101.2473688760708, Tokens per sec: 82731.83144777875, Loss: 2.331303358078003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3983 , TFLOPS: 101.42989616378374, Tokens per sec: 82880.97919323937, Loss: 2.335394859313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3984 , TFLOPS: 102.93356340407138, Tokens per sec: 84109.66440311672, Loss: 2.307229995727539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3985 , TFLOPS: 100.98174668050686, Tokens per sec: 82514.78471405983, Loss: 2.2972006797790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3986 , TFLOPS: 100.73217718141075, Tokens per sec: 82310.85505185796, Loss: 2.312598705291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3987 , TFLOPS: 100.38550593998883, Tokens per sec: 82027.58105638046, Loss: 2.307025909423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3988 , TFLOPS: 101.65810930618385, Tokens per sec: 83067.45802662343, Loss: 2.325235605239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3989 , TFLOPS: 101.64787517539041, Tokens per sec: 83059.09545490214, Loss: 2.3301949501037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3990 , TFLOPS: 98.39854926077108, Tokens per sec: 80403.98760497637, Loss: 2.3154866695404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3991 , TFLOPS: 100.50583280350133, Tokens per sec: 82125.90323404555, Loss: 2.3017654418945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3992 , TFLOPS: 101.18783475123881, Tokens per sec: 82683.18458183382, Loss: 2.319579839706421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3993 , TFLOPS: 101.99147166348699, Tokens per sec: 83339.85699028634, Loss: 2.338758945465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3994 , TFLOPS: 102.26081953744453, Tokens per sec: 83559.94807172815, Loss: 2.318537712097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3995 , TFLOPS: 102.8073917595747, Tokens per sec: 84006.56630445141, Loss: 2.3195443153381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3996 , TFLOPS: 102.71310072511638, Tokens per sec: 83929.51867292836, Loss: 2.3166699409484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3997 , TFLOPS: 102.87567179457801, Tokens per sec: 84062.35967873708, Loss: 2.3226733207702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3998 , TFLOPS: 102.70513134524282, Tokens per sec: 83923.00668748358, Loss: 2.319051504135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3999 , TFLOPS: 102.62074407465504, Tokens per sec: 83854.0516763647, Loss: 2.325312376022339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4000 , TFLOPS: 100.18388927658988, Tokens per sec: 81862.83489064252, Loss: 2.3432984352111816 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/optimizer.pt +[rank0]:[2024-08-27 15:55:12,523] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007685883989324793, 'preprocessing_with_comm': 0.0015524699992965907, 'state_converting': 3.2905216460058, : 3.3014420800027438}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000, took 15.85s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002000 + eval ppl=7.507927894592285, eval loss=2.0159595012664795 +------------------------------------------------------------------ +iteration: 4001 , TFLOPS: 101.36070940146708, Tokens per sec: 82824.44490872473, Loss: 2.3284127712249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4002 , TFLOPS: 100.75836460640286, Tokens per sec: 82332.25347094331, Loss: 2.317943811416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4003 , TFLOPS: 102.82791829755142, Tokens per sec: 84023.3390670323, Loss: 2.310425043106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4004 , TFLOPS: 100.79787258268676, Tokens per sec: 82364.53645539067, Loss: 2.3150386810302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4005 , TFLOPS: 100.38756979937168, Tokens per sec: 82029.26748901064, Loss: 2.316197633743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4006 , TFLOPS: 103.37023171214534, Tokens per sec: 84466.47731848621, Loss: 2.3358938694000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4007 , TFLOPS: 102.14765931044546, Tokens per sec: 83467.48193724382, Loss: 2.3226711750030518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4008 , TFLOPS: 101.74156856031789, Tokens per sec: 83135.65473160868, Loss: 2.3256585597991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4009 , TFLOPS: 102.65065922441421, Tokens per sec: 83878.4961153174, Loss: 2.3050999641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4010 , TFLOPS: 103.36985647305076, Tokens per sec: 84466.17070096252, Loss: 2.3105430603027344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4011 , TFLOPS: 103.26673399177957, Tokens per sec: 84381.90666690702, Loss: 2.310582399368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4012 , TFLOPS: 102.12195193353791, Tokens per sec: 83446.4757778059, Loss: 2.321560859680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4013 , TFLOPS: 101.9227609250328, Tokens per sec: 83283.71167712366, Loss: 2.3152170181274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4014 , TFLOPS: 102.50839454664018, Tokens per sec: 83762.24798488963, Loss: 2.313016176223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4015 , TFLOPS: 101.25854957741004, Tokens per sec: 82740.9674866599, Loss: 2.3181281089782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4016 , TFLOPS: 101.95169759710673, Tokens per sec: 83307.35657677148, Loss: 2.3301830291748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4017 , TFLOPS: 102.698657209107, Tokens per sec: 83917.7165041879, Loss: 2.3086419105529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4018 , TFLOPS: 102.51696077150432, Tokens per sec: 83769.24766774035, Loss: 2.3185439109802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4019 , TFLOPS: 101.30169972612296, Tokens per sec: 82776.22658395687, Loss: 2.3134207725524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4020 , TFLOPS: 102.56928495253186, Tokens per sec: 83812.00310300221, Loss: 2.302018165588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4021 , TFLOPS: 100.76853850666781, Tokens per sec: 82340.56682674929, Loss: 2.3175480365753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4022 , TFLOPS: 102.48826027753698, Tokens per sec: 83745.7957553032, Loss: 2.28987455368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4023 , TFLOPS: 101.4367164645854, Tokens per sec: 82886.55223659458, Loss: 2.3297674655914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4024 , TFLOPS: 101.34862713839061, Tokens per sec: 82814.57218053982, Loss: 2.331112861633301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4025 , TFLOPS: 101.67600839732292, Tokens per sec: 83082.08383475676, Loss: 2.312580108642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4026 , TFLOPS: 102.37755067619098, Tokens per sec: 83655.3320901247, Loss: 2.3120274543762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4027 , TFLOPS: 101.45217736030855, Tokens per sec: 82899.18573248883, Loss: 2.3304293155670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4028 , TFLOPS: 102.33111890030831, Tokens per sec: 83617.39149079073, Loss: 2.3150572776794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4029 , TFLOPS: 100.82929644607867, Tokens per sec: 82390.2137030904, Loss: 2.312762975692749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4030 , TFLOPS: 101.40412305301626, Tokens per sec: 82859.91932097237, Loss: 2.2894721031188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4031 , TFLOPS: 103.351984087281, Tokens per sec: 84451.56671447391, Loss: 2.335055351257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4032 , TFLOPS: 100.88391149591362, Tokens per sec: 82434.841066227, Loss: 2.3323755264282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4033 , TFLOPS: 102.73802467419306, Tokens per sec: 83949.88467331845, Loss: 2.3274457454681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4034 , TFLOPS: 100.77193999920732, Tokens per sec: 82343.34627386552, Loss: 2.299907922744751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4035 , TFLOPS: 102.53650581401672, Tokens per sec: 83785.2184251114, Loss: 2.3168444633483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4036 , TFLOPS: 100.89215305634609, Tokens per sec: 82441.57545741301, Loss: 2.303889513015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4037 , TFLOPS: 101.96952630979517, Tokens per sec: 83321.92487686109, Loss: 2.318115234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4038 , TFLOPS: 101.50514466414937, Tokens per sec: 82942.46667995687, Loss: 2.3148529529571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4039 , TFLOPS: 102.66207376350475, Tokens per sec: 83887.82322904465, Loss: 2.3367419242858887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4040 , TFLOPS: 100.92808701359068, Tokens per sec: 82470.93801889992, Loss: 2.328162908554077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4041 , TFLOPS: 102.0816602516194, Tokens per sec: 83413.55240731013, Loss: 2.2898221015930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4042 , TFLOPS: 100.85671580384376, Tokens per sec: 82412.61876615732, Loss: 2.305424928665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4043 , TFLOPS: 100.3478612914184, Tokens per sec: 81996.82064497408, Loss: 2.319782018661499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4044 , TFLOPS: 103.42605905669626, Tokens per sec: 84512.09527884248, Loss: 2.3122501373291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4045 , TFLOPS: 101.67533109251639, Tokens per sec: 83081.53039156397, Loss: 2.317996025085449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4046 , TFLOPS: 102.78425327863614, Tokens per sec: 83987.65925603903, Loss: 2.307156562805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4047 , TFLOPS: 103.39078215519346, Tokens per sec: 84483.26961451632, Loss: 2.3250551223754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4048 , TFLOPS: 102.71462423923214, Tokens per sec: 83930.76357553108, Loss: 2.3246541023254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4049 , TFLOPS: 102.69088573728122, Tokens per sec: 83911.36623450358, Loss: 2.3267083168029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4050 , TFLOPS: 101.84639802589939, Tokens per sec: 83221.31358648585, Loss: 2.2984557151794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4051 , TFLOPS: 102.7251849164326, Tokens per sec: 83939.3929767276, Loss: 2.3141837120056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4052 , TFLOPS: 102.67676933337832, Tokens per sec: 83899.83135748595, Loss: 2.323977470397949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4053 , TFLOPS: 100.70820865118625, Tokens per sec: 82291.26974880668, Loss: 2.335196018218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4054 , TFLOPS: 102.45119847929959, Tokens per sec: 83715.51160590787, Loss: 2.344594955444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4055 , TFLOPS: 102.12964306988158, Tokens per sec: 83452.76040329978, Loss: 2.282050132751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4056 , TFLOPS: 102.74347489931549, Tokens per sec: 83954.33819256726, Loss: 2.324990749359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4057 , TFLOPS: 102.05213085348, Tokens per sec: 83389.42317593608, Loss: 2.3315463066101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4058 , TFLOPS: 101.34687103153846, Tokens per sec: 82813.13722042473, Loss: 2.303511142730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4059 , TFLOPS: 101.50113417345199, Tokens per sec: 82939.18960476882, Loss: 2.32326078414917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4060 , TFLOPS: 102.93534155770226, Tokens per sec: 84111.11737822226, Loss: 2.3074185848236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4061 , TFLOPS: 100.77433059085004, Tokens per sec: 82345.2996878361, Loss: 2.3193893432617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4062 , TFLOPS: 102.51909333137839, Tokens per sec: 83770.99023731043, Loss: 2.3348569869995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4063 , TFLOPS: 101.62463261299186, Tokens per sec: 83040.1033588495, Loss: 2.3127188682556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4064 , TFLOPS: 102.08640738691649, Tokens per sec: 83417.43141376352, Loss: 2.3309056758880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4065 , TFLOPS: 102.02790729855653, Tokens per sec: 83369.62948563833, Loss: 2.3249809741973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4066 , TFLOPS: 102.12666895425096, Tokens per sec: 83450.33017685746, Loss: 2.2998275756835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4067 , TFLOPS: 102.1886216170039, Tokens per sec: 83500.9532923963, Loss: 2.314974784851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4068 , TFLOPS: 101.4203087363094, Tokens per sec: 82873.14505944776, Loss: 2.3113670349121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4069 , TFLOPS: 102.70053321406857, Tokens per sec: 83919.24943613469, Loss: 2.302147388458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4070 , TFLOPS: 100.80644462519648, Tokens per sec: 82371.54089198937, Loss: 2.33825421333313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4071 , TFLOPS: 102.6169676808977, Tokens per sec: 83850.96588781256, Loss: 2.316375494003296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4072 , TFLOPS: 100.89255887598998, Tokens per sec: 82441.90706309075, Loss: 2.3006539344787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4073 , TFLOPS: 102.00625974424271, Tokens per sec: 83351.9407117509, Loss: 2.312056064605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4074 , TFLOPS: 100.01892307863575, Tokens per sec: 81728.03676368648, Loss: 2.3032729625701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4075 , TFLOPS: 102.01195068192986, Tokens per sec: 83356.59092343284, Loss: 2.3073647022247314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4076 , TFLOPS: 101.5722187765837, Tokens per sec: 82997.27466386821, Loss: 2.2963826656341553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4077 , TFLOPS: 102.20482846722693, Tokens per sec: 83514.19632691497, Loss: 2.3113527297973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4078 , TFLOPS: 101.00108618031697, Tokens per sec: 82530.5875171977, Loss: 2.3400232791900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4079 , TFLOPS: 102.8941187468557, Tokens per sec: 84077.43315830998, Loss: 2.2893927097320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4080 , TFLOPS: 100.31042803840037, Tokens per sec: 81966.23297031541, Loss: 2.293639898300171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4081 , TFLOPS: 101.1565327727292, Tokens per sec: 82657.606929409, Loss: 2.316769599914551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4082 , TFLOPS: 102.74020395325215, Tokens per sec: 83951.66541833724, Loss: 2.3261499404907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4083 , TFLOPS: 102.02081068451272, Tokens per sec: 83363.83065962004, Loss: 2.3171823024749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4084 , TFLOPS: 101.91078443600793, Tokens per sec: 83273.92537963917, Loss: 2.307715892791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4085 , TFLOPS: 102.85703865248173, Tokens per sec: 84047.13405867048, Loss: 2.313715934753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4086 , TFLOPS: 103.35391583134337, Tokens per sec: 84453.1451922748, Loss: 2.315506935119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4087 , TFLOPS: 102.58567881652588, Tokens per sec: 83825.39895128732, Loss: 2.3105413913726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4088 , TFLOPS: 102.21231833360768, Tokens per sec: 83520.316489541, Loss: 2.305522918701172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4089 , TFLOPS: 102.70037131451784, Tokens per sec: 83919.11714384411, Loss: 2.3304715156555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4090 , TFLOPS: 102.10638550640259, Tokens per sec: 83433.75604947815, Loss: 2.316226005554199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4091 , TFLOPS: 100.64615010072406, Tokens per sec: 82240.5601096948, Loss: 2.321143627166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4092 , TFLOPS: 102.88193862051789, Tokens per sec: 84067.480463535, Loss: 2.303358316421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4093 , TFLOPS: 101.5818980509808, Tokens per sec: 83005.18384814478, Loss: 2.30267333984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4094 , TFLOPS: 103.37074577759066, Tokens per sec: 84466.89737459482, Loss: 2.2983806133270264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4095 , TFLOPS: 101.94789795251774, Tokens per sec: 83304.25178936624, Loss: 2.3212697505950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4096 , TFLOPS: 102.58613766775943, Tokens per sec: 83825.77389044214, Loss: 2.3147659301757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4097 , TFLOPS: 100.94891703128728, Tokens per sec: 82487.95876257218, Loss: 2.292545795440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4098 , TFLOPS: 103.3938800964481, Tokens per sec: 84485.80102206358, Loss: 2.305776834487915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4099 , TFLOPS: 101.91160670277971, Tokens per sec: 83274.59727498543, Loss: 2.3192038536071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4100 , TFLOPS: 101.92929163759806, Tokens per sec: 83289.04809047622, Loss: 2.3239643573760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4101 , TFLOPS: 100.95740760574357, Tokens per sec: 82494.89662952723, Loss: 2.324207305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4102 , TFLOPS: 101.44697748910059, Tokens per sec: 82894.93678386821, Loss: 2.303504228591919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4103 , TFLOPS: 102.68228774370225, Tokens per sec: 83904.34059261816, Loss: 2.3207318782806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4104 , TFLOPS: 102.28609014531358, Tokens per sec: 83580.59733594127, Loss: 2.324038028717041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4105 , TFLOPS: 101.0527894279997, Tokens per sec: 82572.83557183924, Loss: 2.308579444885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4106 , TFLOPS: 102.01350527307652, Tokens per sec: 83357.86121987751, Loss: 2.3260905742645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4107 , TFLOPS: 102.64505197201412, Tokens per sec: 83873.91428503775, Loss: 2.2915775775909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4108 , TFLOPS: 101.27528976913881, Tokens per sec: 82754.64632825236, Loss: 2.3238861560821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4109 , TFLOPS: 102.00980664661618, Tokens per sec: 83354.83897698564, Loss: 2.3226752281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4110 , TFLOPS: 100.92041528058662, Tokens per sec: 82464.66924837432, Loss: 2.292433500289917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4111 , TFLOPS: 102.11380018501102, Tokens per sec: 83439.8147742399, Loss: 2.3333559036254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4112 , TFLOPS: 100.16111899596766, Tokens per sec: 81844.22870818693, Loss: 2.330037832260132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4113 , TFLOPS: 101.23827157196055, Tokens per sec: 82724.39780640483, Loss: 2.327260971069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4114 , TFLOPS: 101.59490766946841, Tokens per sec: 83015.81434230838, Loss: 2.315366744995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4115 , TFLOPS: 102.07898625010114, Tokens per sec: 83411.36741183464, Loss: 2.320415496826172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4116 , TFLOPS: 101.311968265487, Tokens per sec: 82784.61727180694, Loss: 2.3057987689971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4117 , TFLOPS: 102.03670330364831, Tokens per sec: 83376.81692782816, Loss: 2.3052480220794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4118 , TFLOPS: 100.89893549371197, Tokens per sec: 82447.11756158015, Loss: 2.3108935356140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4119 , TFLOPS: 101.12190939535171, Tokens per sec: 82629.31527646884, Loss: 2.3115572929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4120 , TFLOPS: 102.77676129953176, Tokens per sec: 83981.53736705214, Loss: 2.3193681240081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4121 , TFLOPS: 101.57962700409318, Tokens per sec: 83003.32811727095, Loss: 2.301459789276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4122 , TFLOPS: 102.05918846487661, Tokens per sec: 83395.19013188734, Loss: 2.324505090713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4123 , TFLOPS: 103.45673147770128, Tokens per sec: 84537.15850362404, Loss: 2.3109054565429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4124 , TFLOPS: 102.83612605102665, Tokens per sec: 84030.04583368327, Loss: 2.3164029121398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4125 , TFLOPS: 102.1206091052203, Tokens per sec: 83445.37851822011, Loss: 2.2970731258392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4126 , TFLOPS: 102.00921105705929, Tokens per sec: 83354.35230542689, Loss: 2.3029837608337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4127 , TFLOPS: 103.38158356729184, Tokens per sec: 84475.75321154918, Loss: 2.2825701236724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4128 , TFLOPS: 102.17662010057865, Tokens per sec: 83491.1465443787, Loss: 2.313277244567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4129 , TFLOPS: 101.40579199468999, Tokens per sec: 82861.28305617637, Loss: 2.3060996532440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4130 , TFLOPS: 103.39161112806656, Tokens per sec: 84483.94698959058, Loss: 2.308570623397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4131 , TFLOPS: 102.02586539312641, Tokens per sec: 83367.96099214807, Loss: 2.32484769821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4132 , TFLOPS: 102.59788235255354, Tokens per sec: 83835.37077472248, Loss: 2.312046766281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4133 , TFLOPS: 102.1111642157293, Tokens per sec: 83437.66085588394, Loss: 2.3019065856933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4134 , TFLOPS: 102.02296483492633, Tokens per sec: 83365.59087137577, Loss: 2.3117198944091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4135 , TFLOPS: 101.49839331323335, Tokens per sec: 82936.94997732814, Loss: 2.3233635425567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4136 , TFLOPS: 102.64131835429049, Tokens per sec: 83870.86344988391, Loss: 2.316939353942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4137 , TFLOPS: 101.77281390542875, Tokens per sec: 83161.18610742551, Loss: 2.3081436157226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4138 , TFLOPS: 101.64736820572718, Tokens per sec: 83058.68119693975, Loss: 2.2937240600585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4139 , TFLOPS: 101.00252890245258, Tokens per sec: 82531.76640259367, Loss: 2.298144817352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4140 , TFLOPS: 102.29205879778144, Tokens per sec: 83585.47447551953, Loss: 2.301252603530884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4141 , TFLOPS: 102.06827960264752, Tokens per sec: 83402.61873458666, Loss: 2.3263394832611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4142 , TFLOPS: 102.23025300019492, Tokens per sec: 83534.97136728883, Loss: 2.3081886768341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4143 , TFLOPS: 101.1592042904071, Tokens per sec: 82659.7898952746, Loss: 2.318812847137451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4144 , TFLOPS: 101.87167715669835, Tokens per sec: 83241.9698150044, Loss: 2.3220462799072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4145 , TFLOPS: 102.65832120535457, Tokens per sec: 83884.75691718068, Loss: 2.2996602058410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4146 , TFLOPS: 102.06982298577616, Tokens per sec: 83403.87987267127, Loss: 2.3022546768188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4147 , TFLOPS: 101.48045440229686, Tokens per sec: 82922.2916314133, Loss: 2.3201987743377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4148 , TFLOPS: 102.06723719561369, Tokens per sec: 83401.76695696537, Loss: 2.3041882514953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4149 , TFLOPS: 102.52414927705289, Tokens per sec: 83775.12158067268, Loss: 2.3070828914642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4150 , TFLOPS: 100.78270904601627, Tokens per sec: 82352.14593923306, Loss: 2.320225954055786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4151 , TFLOPS: 102.05755327437916, Tokens per sec: 83393.85397563833, Loss: 2.3031163215637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4152 , TFLOPS: 101.58868661153963, Tokens per sec: 83010.7309557305, Loss: 2.326996088027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4153 , TFLOPS: 101.2673349589736, Tokens per sec: 82748.14624809133, Loss: 2.3180556297302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4154 , TFLOPS: 101.37419101294117, Tokens per sec: 82835.46108050777, Loss: 2.316281795501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4155 , TFLOPS: 102.15492923092303, Tokens per sec: 83473.42237641044, Loss: 2.2992477416992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4156 , TFLOPS: 100.94096451874418, Tokens per sec: 82481.46055985733, Loss: 2.3049428462982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4157 , TFLOPS: 100.10214628282704, Tokens per sec: 81796.04058618705, Loss: 2.289364814758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4158 , TFLOPS: 102.02011056282291, Tokens per sec: 83363.25857216456, Loss: 2.3319108486175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4159 , TFLOPS: 102.34087022321202, Tokens per sec: 83625.35954775676, Loss: 2.308746099472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4160 , TFLOPS: 101.57410468716125, Tokens per sec: 82998.8156899488, Loss: 2.318214178085327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4161 , TFLOPS: 103.35801229777411, Tokens per sec: 84456.49252044778, Loss: 2.312788963317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4162 , TFLOPS: 102.9447318970462, Tokens per sec: 84118.79046622904, Loss: 2.3199949264526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4163 , TFLOPS: 102.22155652519052, Tokens per sec: 83527.86525369555, Loss: 2.3295795917510986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4164 , TFLOPS: 102.75830815444657, Tokens per sec: 83966.45882717682, Loss: 2.322240114212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4165 , TFLOPS: 103.35383259073076, Tokens per sec: 84453.07717422742, Loss: 2.3111581802368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4166 , TFLOPS: 101.99578197708135, Tokens per sec: 83343.37905848163, Loss: 2.310802936553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4167 , TFLOPS: 100.77869165231422, Tokens per sec: 82348.8632234222, Loss: 2.3261821269989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4168 , TFLOPS: 102.64215089675386, Tokens per sec: 83871.54374176227, Loss: 2.3210058212280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4169 , TFLOPS: 102.6352655811713, Tokens per sec: 83865.91757315597, Loss: 2.3038437366485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4170 , TFLOPS: 102.75014833142222, Tokens per sec: 83959.7912257311, Loss: 2.324021577835083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4171 , TFLOPS: 102.17285950246062, Tokens per sec: 83488.07366285006, Loss: 2.3275604248046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4172 , TFLOPS: 101.95115677809295, Tokens per sec: 83306.91465963339, Loss: 2.3244481086730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4173 , TFLOPS: 100.83400750192142, Tokens per sec: 82394.06322809288, Loss: 2.329552173614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4174 , TFLOPS: 102.47007182222637, Tokens per sec: 83730.9335002564, Loss: 2.3126323223114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4175 , TFLOPS: 102.1815443703503, Tokens per sec: 83495.1702919711, Loss: 2.3204185962677 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4176 , TFLOPS: 101.92329048880401, Tokens per sec: 83284.1443973133, Loss: 2.3210055828094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4177 , TFLOPS: 100.98877404748002, Tokens per sec: 82520.52695651412, Loss: 2.322826623916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4178 , TFLOPS: 102.07683471740086, Tokens per sec: 83409.60933907985, Loss: 2.323307514190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4179 , TFLOPS: 102.72881619632392, Tokens per sec: 83942.36018900425, Loss: 2.3266825675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4180 , TFLOPS: 102.84344961414978, Tokens per sec: 84036.03010563646, Loss: 2.343097448348999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4181 , TFLOPS: 101.59056037454295, Tokens per sec: 83012.26205571544, Loss: 2.2979397773742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4182 , TFLOPS: 101.93733369013849, Tokens per sec: 83295.61945862786, Loss: 2.336824417114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4183 , TFLOPS: 103.29079295297329, Tokens per sec: 84401.56586343115, Loss: 2.307917356491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4184 , TFLOPS: 102.2209288042342, Tokens per sec: 83527.35232674316, Loss: 2.3063995838165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4185 , TFLOPS: 100.289586933057, Tokens per sec: 81949.20316664154, Loss: 2.317274332046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4186 , TFLOPS: 102.01977283170177, Tokens per sec: 83362.98260337152, Loss: 2.3261613845825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4187 , TFLOPS: 102.60360173073695, Tokens per sec: 83840.04422586605, Loss: 2.319453716278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4188 , TFLOPS: 100.23606685292992, Tokens per sec: 81905.47053144028, Loss: 2.3208162784576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4189 , TFLOPS: 102.6791297387805, Tokens per sec: 83901.76010550233, Loss: 2.3120265007019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4190 , TFLOPS: 100.7289550388167, Tokens per sec: 82308.22215619903, Loss: 2.3063831329345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4191 , TFLOPS: 101.94051513265427, Tokens per sec: 83298.21909720512, Loss: 2.31014084815979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4192 , TFLOPS: 101.60309352327295, Tokens per sec: 83022.50321417468, Loss: 2.3034353256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4193 , TFLOPS: 102.0000014723403, Tokens per sec: 83346.82691667718, Loss: 2.3035616874694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4194 , TFLOPS: 100.9130851573314, Tokens per sec: 82458.67961596831, Loss: 2.307666540145874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4195 , TFLOPS: 99.84218578745872, Tokens per sec: 81583.62017344311, Loss: 2.309455633163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4196 , TFLOPS: 101.95251724195977, Tokens per sec: 83308.02632968043, Loss: 2.319589853286743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4197 , TFLOPS: 102.35989751269929, Tokens per sec: 83640.9072358028, Loss: 2.2992279529571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4198 , TFLOPS: 100.90893835879334, Tokens per sec: 82455.29116013479, Loss: 2.321362257003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4199 , TFLOPS: 103.31265934842375, Tokens per sec: 84419.43345804482, Loss: 2.313326835632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4200 , TFLOPS: 102.05793043779686, Tokens per sec: 83394.16216557598, Loss: 2.3281116485595703 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/optimizer.pt +[rank0]:[2024-08-27 16:48:03,424] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0077902649936731905, 'preprocessing_with_comm': 0.0015279630024451762, 'state_converting': 3.3947779060108587, : 3.4056797040102538}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200, took 15.92s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002200 + eval ppl=8.481754302978516, eval loss=2.1379172801971436 +------------------------------------------------------------------ +iteration: 4201 , TFLOPS: 100.82209623787075, Tokens per sec: 82384.33022761386, Loss: 2.309843063354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4202 , TFLOPS: 102.60505827260604, Tokens per sec: 83841.23440372202, Loss: 2.327726125717163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4203 , TFLOPS: 99.68861571966667, Tokens per sec: 81458.13411780499, Loss: 2.303619146347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4204 , TFLOPS: 103.37749149535469, Tokens per sec: 84472.40947423002, Loss: 2.3204243183135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4205 , TFLOPS: 100.7747318450998, Tokens per sec: 82345.62756301278, Loss: 2.2843244075775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4206 , TFLOPS: 102.82065776620743, Tokens per sec: 84017.40629996842, Loss: 2.3196396827697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4207 , TFLOPS: 101.33940737368023, Tokens per sec: 82807.03847345713, Loss: 2.298419713973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4208 , TFLOPS: 101.48454728699473, Tokens per sec: 82925.63603286014, Loss: 2.2934012413024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4209 , TFLOPS: 101.72587398252814, Tokens per sec: 83122.83028808232, Loss: 2.304072618484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4210 , TFLOPS: 102.82214802727368, Tokens per sec: 84018.62403064838, Loss: 2.313331365585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4211 , TFLOPS: 101.94328340540352, Tokens per sec: 83300.48112413022, Loss: 2.313082456588745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4212 , TFLOPS: 103.34444070706141, Tokens per sec: 84445.40282431261, Loss: 2.290980339050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4213 , TFLOPS: 102.54460187470666, Tokens per sec: 83791.83392471228, Loss: 2.3019373416900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4214 , TFLOPS: 103.03393629024359, Tokens per sec: 84191.681672236, Loss: 2.301682472229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4215 , TFLOPS: 101.53089974299643, Tokens per sec: 82963.5118178775, Loss: 2.3121044635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4216 , TFLOPS: 101.28545033073992, Tokens per sec: 82762.94878469274, Loss: 2.294565439224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4217 , TFLOPS: 102.58993795342839, Tokens per sec: 83828.8792016903, Loss: 2.3021960258483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4218 , TFLOPS: 102.59214762399792, Tokens per sec: 83830.68478039464, Loss: 2.3197150230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4219 , TFLOPS: 102.58785675615685, Tokens per sec: 83827.17860182495, Loss: 2.317671060562134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4220 , TFLOPS: 102.82082800618203, Tokens per sec: 84017.54540743401, Loss: 2.311534881591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4221 , TFLOPS: 102.55743482222164, Tokens per sec: 83802.32005647622, Loss: 2.3129491806030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4222 , TFLOPS: 101.90873816700223, Tokens per sec: 83272.25332055897, Loss: 2.3259100914001465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4223 , TFLOPS: 102.69483606976426, Tokens per sec: 83914.59415286609, Loss: 2.3110454082489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4224 , TFLOPS: 100.1843528044338, Tokens per sec: 81863.21365117606, Loss: 2.303466796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4225 , TFLOPS: 103.28529202209953, Tokens per sec: 84397.07091120764, Loss: 2.2945637702941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4226 , TFLOPS: 101.42784854858488, Tokens per sec: 82879.3060341501, Loss: 2.293863296508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4227 , TFLOPS: 102.04826146805537, Tokens per sec: 83386.26140149878, Loss: 2.2928409576416016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4228 , TFLOPS: 101.99033488507267, Tokens per sec: 83338.9280993807, Loss: 2.3072032928466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4229 , TFLOPS: 102.81683487083046, Tokens per sec: 84014.28251374581, Loss: 2.3119776248931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4230 , TFLOPS: 100.3021421568949, Tokens per sec: 81959.46236323971, Loss: 2.2976388931274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4231 , TFLOPS: 102.6604444751201, Tokens per sec: 83886.49189556393, Loss: 2.3294055461883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4232 , TFLOPS: 101.79093067050272, Tokens per sec: 83175.98978253447, Loss: 2.3172547817230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4233 , TFLOPS: 100.82308551000506, Tokens per sec: 82385.13858734095, Loss: 2.300724744796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4234 , TFLOPS: 102.12816965149855, Tokens per sec: 83451.55643520984, Loss: 2.3039238452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4235 , TFLOPS: 100.09017450940503, Tokens per sec: 81786.25814194306, Loss: 2.3274142742156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4236 , TFLOPS: 102.17928526534828, Tokens per sec: 83493.32431911942, Loss: 2.3214645385742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4237 , TFLOPS: 101.11796331457603, Tokens per sec: 82626.09083228587, Loss: 2.304626226425171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4238 , TFLOPS: 98.68489243005567, Tokens per sec: 80637.96597972774, Loss: 2.2834115028381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4239 , TFLOPS: 101.20340693274701, Tokens per sec: 82695.90901220779, Loss: 2.2975974082946777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4240 , TFLOPS: 102.111659814838, Tokens per sec: 83438.06582267351, Loss: 2.302950143814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4241 , TFLOPS: 100.81756606364573, Tokens per sec: 82380.62850563753, Loss: 2.316251754760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4242 , TFLOPS: 102.57188199805469, Tokens per sec: 83814.12521575282, Loss: 2.2995924949645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4243 , TFLOPS: 101.68501286791097, Tokens per sec: 83089.44161946994, Loss: 2.321830987930298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4244 , TFLOPS: 101.98212588691558, Tokens per sec: 83332.22031566814, Loss: 2.2970008850097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4245 , TFLOPS: 102.35216153256528, Tokens per sec: 83634.58596729345, Loss: 2.3008205890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4246 , TFLOPS: 101.61773749255612, Tokens per sec: 83034.46917844527, Loss: 2.293759346008301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4247 , TFLOPS: 101.93683951637918, Tokens per sec: 83295.21565652796, Loss: 2.2919538021087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4248 , TFLOPS: 102.06654672046834, Tokens per sec: 83401.20275194969, Loss: 2.2852494716644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4249 , TFLOPS: 102.04095526356662, Tokens per sec: 83380.29131373258, Loss: 2.3081769943237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4250 , TFLOPS: 103.37287653111791, Tokens per sec: 84468.6384681524, Loss: 2.3069827556610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4251 , TFLOPS: 102.06634406311116, Tokens per sec: 83401.03715540627, Loss: 2.3352911472320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4252 , TFLOPS: 102.98589363389148, Tokens per sec: 84152.42478100269, Loss: 2.3173749446868896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4253 , TFLOPS: 102.75379086516062, Tokens per sec: 83962.76763381589, Loss: 2.2777099609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4254 , TFLOPS: 101.33281302871885, Tokens per sec: 82801.65006443564, Loss: 2.3076138496398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4255 , TFLOPS: 102.2048610740933, Tokens per sec: 83514.22297082486, Loss: 2.310267210006714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4256 , TFLOPS: 101.27701131467143, Tokens per sec: 82756.05304742357, Loss: 2.322251796722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4257 , TFLOPS: 103.22360205561847, Tokens per sec: 84346.66245155496, Loss: 2.30309796333313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4258 , TFLOPS: 101.36971794885928, Tokens per sec: 82831.80602469954, Loss: 2.314368724822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4259 , TFLOPS: 102.83058451422959, Tokens per sec: 84025.51770131454, Loss: 2.31264591217041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4260 , TFLOPS: 102.00824209710638, Tokens per sec: 83353.5605433061, Loss: 2.3077399730682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4261 , TFLOPS: 102.7086014889759, Tokens per sec: 83925.84223125732, Loss: 2.2941317558288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4262 , TFLOPS: 101.53765521998534, Tokens per sec: 82969.0318920266, Loss: 2.299516201019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4263 , TFLOPS: 102.20039789347756, Tokens per sec: 83510.57599105126, Loss: 2.310007333755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4264 , TFLOPS: 102.67413781576458, Tokens per sec: 83897.68107670249, Loss: 2.3112857341766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4265 , TFLOPS: 100.05851367661629, Tokens per sec: 81760.38726044905, Loss: 2.3005425930023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4266 , TFLOPS: 102.0494231863076, Tokens per sec: 83387.21067138798, Loss: 2.301877975463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4267 , TFLOPS: 102.10975470193642, Tokens per sec: 83436.50910587965, Loss: 2.337308645248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4268 , TFLOPS: 101.58999694323364, Tokens per sec: 83011.80166148861, Loss: 2.312824249267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4269 , TFLOPS: 102.32089777644576, Tokens per sec: 83609.0395473675, Loss: 2.3041088581085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4270 , TFLOPS: 101.29958311561118, Tokens per sec: 82774.49704702129, Loss: 2.3162758350372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4271 , TFLOPS: 100.76083790021059, Tokens per sec: 82334.27446298185, Loss: 2.310497760772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4272 , TFLOPS: 102.11875410347444, Tokens per sec: 83443.86274854158, Loss: 2.302562952041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4273 , TFLOPS: 101.13887094822341, Tokens per sec: 82643.17499794952, Loss: 2.2971177101135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4274 , TFLOPS: 102.54069449224167, Tokens per sec: 83788.64110191517, Loss: 2.281583786010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4275 , TFLOPS: 101.5021997247061, Tokens per sec: 82940.06029412829, Loss: 2.3316195011138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4276 , TFLOPS: 101.58554517068802, Tokens per sec: 83008.16400354276, Loss: 2.2971246242523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4277 , TFLOPS: 101.74297897137714, Tokens per sec: 83136.80721479241, Loss: 2.2873752117156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4278 , TFLOPS: 102.86902756906058, Tokens per sec: 84056.93051102922, Loss: 2.307863712310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4279 , TFLOPS: 100.80342344586406, Tokens per sec: 82369.07220858477, Loss: 2.305241346359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4280 , TFLOPS: 102.01879306006904, Tokens per sec: 83362.18200674884, Loss: 2.3032586574554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4281 , TFLOPS: 102.12357281978332, Tokens per sec: 83447.80024568236, Loss: 2.316145181655884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4282 , TFLOPS: 102.73252354673376, Tokens per sec: 83945.38956045982, Loss: 2.3176536560058594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4283 , TFLOPS: 102.06163016070633, Tokens per sec: 83397.18530440464, Loss: 2.3243408203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4284 , TFLOPS: 101.07541806617657, Tokens per sec: 82591.32601460654, Loss: 2.3198299407958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4285 , TFLOPS: 103.37302109771464, Tokens per sec: 84468.75659724008, Loss: 2.302154779434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4286 , TFLOPS: 100.6950356154658, Tokens per sec: 82280.50572221537, Loss: 2.307511329650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4287 , TFLOPS: 102.35525375623854, Tokens per sec: 83637.11270285779, Loss: 2.307544469833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4288 , TFLOPS: 102.56310398212587, Tokens per sec: 83806.95247297114, Loss: 2.31571626663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4289 , TFLOPS: 103.39888941614217, Tokens per sec: 84489.89426613695, Loss: 2.295844078063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4290 , TFLOPS: 101.43875267175953, Tokens per sec: 82888.2160738932, Loss: 2.330068349838257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4291 , TFLOPS: 102.66637790571069, Tokens per sec: 83891.3402544396, Loss: 2.3122806549072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4292 , TFLOPS: 99.95170984769963, Tokens per sec: 81673.11510246608, Loss: 2.328045129776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4293 , TFLOPS: 102.66514397638527, Tokens per sec: 83890.33197902363, Loss: 2.322735548019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4294 , TFLOPS: 101.54763930153693, Tokens per sec: 82977.19014207553, Loss: 2.306899309158325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4295 , TFLOPS: 102.13412756685484, Tokens per sec: 83456.42480121784, Loss: 2.314661979675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4296 , TFLOPS: 101.47836153044322, Tokens per sec: 82920.58149194637, Loss: 2.3003602027893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4297 , TFLOPS: 101.45873660909739, Tokens per sec: 82904.54546352428, Loss: 2.3194947242736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4298 , TFLOPS: 101.43677653653465, Tokens per sec: 82886.60132293071, Loss: 2.309485673904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4299 , TFLOPS: 102.82115461378731, Tokens per sec: 84017.81228691591, Loss: 2.289184331893921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4300 , TFLOPS: 100.27566346354382, Tokens per sec: 81937.82594128035, Loss: 2.3305587768554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4301 , TFLOPS: 102.69012372242692, Tokens per sec: 83910.74357255013, Loss: 2.3177034854888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4302 , TFLOPS: 101.21934542278827, Tokens per sec: 82708.93276271374, Loss: 2.283179998397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4303 , TFLOPS: 101.74491893888073, Tokens per sec: 83138.3924121788, Loss: 2.308821201324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4304 , TFLOPS: 101.60186418336271, Tokens per sec: 83021.49868888801, Loss: 2.315242052078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4305 , TFLOPS: 102.12769020998137, Tokens per sec: 83451.16467120423, Loss: 2.297051429748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4306 , TFLOPS: 101.3669895378548, Tokens per sec: 82829.57656983224, Loss: 2.3223166465759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4307 , TFLOPS: 101.54758713256645, Tokens per sec: 82977.1475134668, Loss: 2.3349339962005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4308 , TFLOPS: 102.74914637361029, Tokens per sec: 83958.97250021026, Loss: 2.3152172565460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4309 , TFLOPS: 100.99201256694268, Tokens per sec: 82523.17323413398, Loss: 2.3030402660369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4310 , TFLOPS: 102.79893134901059, Tokens per sec: 83999.65308518914, Loss: 2.3088951110839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4311 , TFLOPS: 100.28873890929549, Tokens per sec: 81948.51022459594, Loss: 2.306223154067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4312 , TFLOPS: 102.03016940106828, Tokens per sec: 83371.4779078324, Loss: 2.330961227416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4313 , TFLOPS: 101.14961389813779, Tokens per sec: 82651.95333887268, Loss: 2.3157687187194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4314 , TFLOPS: 101.54961718963455, Tokens per sec: 82978.80632535544, Loss: 2.3194944858551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4315 , TFLOPS: 102.3363045064419, Tokens per sec: 83621.62878305187, Loss: 2.3189752101898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4316 , TFLOPS: 101.48434975647825, Tokens per sec: 82925.47462559024, Loss: 2.302947759628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4317 , TFLOPS: 101.48608217587028, Tokens per sec: 82926.89023007192, Loss: 2.3209152221679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4318 , TFLOPS: 101.77212498895187, Tokens per sec: 83160.62317603796, Loss: 2.3040084838867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4319 , TFLOPS: 101.65465934440498, Tokens per sec: 83064.63897404399, Loss: 2.3157296180725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4320 , TFLOPS: 102.07481686835884, Tokens per sec: 83407.96050268356, Loss: 2.300671100616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4321 , TFLOPS: 102.87434291764382, Tokens per sec: 84061.27381918581, Loss: 2.3028173446655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4322 , TFLOPS: 100.5372206804509, Tokens per sec: 82151.55107629686, Loss: 2.3147192001342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4323 , TFLOPS: 102.76357916170589, Tokens per sec: 83970.76590289621, Loss: 2.3178772926330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4324 , TFLOPS: 101.36429262309125, Tokens per sec: 82827.37285135417, Loss: 2.300173044204712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4325 , TFLOPS: 101.43654575969266, Tokens per sec: 82886.41274923239, Loss: 2.321409225463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4326 , TFLOPS: 102.68899327833148, Tokens per sec: 83909.81985757952, Loss: 2.3417134284973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4327 , TFLOPS: 102.05355151606419, Tokens per sec: 83390.58403590473, Loss: 2.30757737159729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4328 , TFLOPS: 102.6109180153601, Tokens per sec: 83846.02255036954, Loss: 2.2843711376190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4329 , TFLOPS: 101.54877568680831, Tokens per sec: 82978.11871173394, Loss: 2.2984631061553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4330 , TFLOPS: 100.71053390338199, Tokens per sec: 82293.16977223312, Loss: 2.3038370609283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4331 , TFLOPS: 101.44708836965852, Tokens per sec: 82895.02738722641, Loss: 2.317375659942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4332 , TFLOPS: 101.59355094099283, Tokens per sec: 83014.70572454568, Loss: 2.3098514080047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4333 , TFLOPS: 102.78104673318619, Tokens per sec: 83985.03910520805, Loss: 2.335376262664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4334 , TFLOPS: 101.28714274725641, Tokens per sec: 82764.33170179505, Loss: 2.2976765632629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4335 , TFLOPS: 101.97214343725831, Tokens per sec: 83324.06339908208, Loss: 2.319551467895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4336 , TFLOPS: 101.40368619724241, Tokens per sec: 82859.5623548739, Loss: 2.325859308242798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4337 , TFLOPS: 102.79698279454938, Tokens per sec: 83998.06087117862, Loss: 2.3067970275878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4338 , TFLOPS: 100.90725085243514, Tokens per sec: 82453.91225524922, Loss: 2.302511692047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4339 , TFLOPS: 102.80713148028833, Tokens per sec: 84006.35362354634, Loss: 2.286689281463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4340 , TFLOPS: 102.2239568468138, Tokens per sec: 83529.82661828371, Loss: 2.3139700889587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4341 , TFLOPS: 100.38096869400067, Tokens per sec: 82023.87355588442, Loss: 2.32078218460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4342 , TFLOPS: 102.13796296943308, Tokens per sec: 83459.55880739658, Loss: 2.3043041229248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4343 , TFLOPS: 101.46541923868337, Tokens per sec: 82910.00601218517, Loss: 2.314500570297241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4344 , TFLOPS: 102.7195594913811, Tokens per sec: 83934.79629710669, Loss: 2.316420555114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4345 , TFLOPS: 102.21632424343977, Tokens per sec: 83523.5898215861, Loss: 2.3267340660095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4346 , TFLOPS: 102.80944908373228, Tokens per sec: 84008.2473969808, Loss: 2.3250505924224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4347 , TFLOPS: 100.85313136929453, Tokens per sec: 82409.68983241559, Loss: 2.3200056552886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4348 , TFLOPS: 102.63469197838056, Tokens per sec: 83865.44886754981, Loss: 2.3154592514038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4349 , TFLOPS: 100.73972005110681, Tokens per sec: 82317.01852485735, Loss: 2.315173864364624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4350 , TFLOPS: 101.98594270385354, Tokens per sec: 83335.33913504177, Loss: 2.323984146118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4351 , TFLOPS: 100.7395895313438, Tokens per sec: 82316.91187379912, Loss: 2.3157835006713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4352 , TFLOPS: 101.50459474414151, Tokens per sec: 82942.01732616225, Loss: 2.3185231685638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4353 , TFLOPS: 102.23769745027596, Tokens per sec: 83541.05441908723, Loss: 2.3162083625793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4354 , TFLOPS: 101.75604495516569, Tokens per sec: 83147.48376649438, Loss: 2.293090343475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4355 , TFLOPS: 101.41836300529387, Tokens per sec: 82871.55515254739, Loss: 2.304361343383789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4356 , TFLOPS: 102.004261474948, Tokens per sec: 83350.30787447016, Loss: 2.298281192779541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4357 , TFLOPS: 101.63113567352295, Tokens per sec: 83045.41717701312, Loss: 2.2642831802368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4358 , TFLOPS: 102.91854453277107, Tokens per sec: 84097.3920967571, Loss: 2.304285764694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4359 , TFLOPS: 101.44873196474256, Tokens per sec: 82896.3704110795, Loss: 2.321369171142578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4360 , TFLOPS: 101.11493783806907, Tokens per sec: 82623.61863754933, Loss: 2.319613218307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4361 , TFLOPS: 102.72130087681356, Tokens per sec: 83936.21922797081, Loss: 2.284369945526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4362 , TFLOPS: 100.82225730292502, Tokens per sec: 82384.46183801588, Loss: 2.3220431804656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4363 , TFLOPS: 102.25286963242502, Tokens per sec: 83553.45199968794, Loss: 2.2945690155029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4364 , TFLOPS: 101.5665969616915, Tokens per sec: 82992.6809342014, Loss: 2.2929787635803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4365 , TFLOPS: 103.41289995529736, Tokens per sec: 84501.34263834402, Loss: 2.2966346740722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4366 , TFLOPS: 102.22112549663943, Tokens per sec: 83527.5130491708, Loss: 2.3123130798339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4367 , TFLOPS: 101.44662715241613, Tokens per sec: 82894.65051474488, Loss: 2.3087873458862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4368 , TFLOPS: 100.86760990246351, Tokens per sec: 82421.52061457856, Loss: 2.2853267192840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4369 , TFLOPS: 102.01600113598413, Tokens per sec: 83359.90065370854, Loss: 2.294626235961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4370 , TFLOPS: 102.19302528919224, Tokens per sec: 83504.55164630199, Loss: 2.3151557445526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4371 , TFLOPS: 102.72917051509432, Tokens per sec: 83942.64971199253, Loss: 2.344515323638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4372 , TFLOPS: 101.116555290465, Tokens per sec: 82624.94029953898, Loss: 2.3041906356811523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4373 , TFLOPS: 102.77081937283799, Tokens per sec: 83976.68206579186, Loss: 2.3181631565093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4374 , TFLOPS: 100.22482250937776, Tokens per sec: 81896.28248886866, Loss: 2.31805682182312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4375 , TFLOPS: 102.07506011084386, Tokens per sec: 83408.1592623797, Loss: 2.320126533508301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4376 , TFLOPS: 101.72852260019158, Tokens per sec: 83124.99454175659, Loss: 2.3166470527648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4377 , TFLOPS: 102.20973953530222, Tokens per sec: 83518.20928706144, Loss: 2.2891814708709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4378 , TFLOPS: 102.23547138447425, Tokens per sec: 83539.23544341663, Loss: 2.2933425903320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4379 , TFLOPS: 101.03586447688407, Tokens per sec: 82559.00575859568, Loss: 2.3056793212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4380 , TFLOPS: 101.51013256389015, Tokens per sec: 82946.54242123502, Loss: 2.308091163635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4381 , TFLOPS: 102.10381823836177, Tokens per sec: 83431.65826867464, Loss: 2.313394546508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4382 , TFLOPS: 102.72011450613232, Tokens per sec: 83935.24981394726, Loss: 2.313295364379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4383 , TFLOPS: 100.25473439460175, Tokens per sec: 81920.72426028561, Loss: 2.31805157661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4384 , TFLOPS: 103.25494201408809, Tokens per sec: 84372.2711383828, Loss: 2.309199571609497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4385 , TFLOPS: 101.0132334242835, Tokens per sec: 82540.51334294061, Loss: 2.293543577194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4386 , TFLOPS: 102.71607260655968, Tokens per sec: 83931.94707375871, Loss: 2.301175117492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4387 , TFLOPS: 100.78777315061902, Tokens per sec: 82356.28394946561, Loss: 2.289139747619629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4388 , TFLOPS: 101.89346650096347, Tokens per sec: 83259.77444911105, Loss: 2.295016288757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4389 , TFLOPS: 99.75164781241571, Tokens per sec: 81509.63926338064, Loss: 2.314821481704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4390 , TFLOPS: 101.59887627100701, Tokens per sec: 83019.05718878645, Loss: 2.293144702911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4391 , TFLOPS: 102.27270633994267, Tokens per sec: 83569.6610840429, Loss: 2.3297152519226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4392 , TFLOPS: 101.654768960848, Tokens per sec: 83064.72854446137, Loss: 2.3026368618011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4393 , TFLOPS: 100.88495169060273, Tokens per sec: 82435.69103608446, Loss: 2.2915151119232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4394 , TFLOPS: 101.97856615450897, Tokens per sec: 83329.31156668303, Loss: 2.2962398529052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4395 , TFLOPS: 102.17769001726396, Tokens per sec: 83492.02080084437, Loss: 2.2969627380371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4396 , TFLOPS: 102.32658533593059, Tokens per sec: 83613.68699863336, Loss: 2.309419870376587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4397 , TFLOPS: 102.62289579146655, Tokens per sec: 83855.8098995615, Loss: 2.298814296722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4398 , TFLOPS: 101.01561048898178, Tokens per sec: 82542.45570369667, Loss: 2.2983555793762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4399 , TFLOPS: 102.30441052238321, Tokens per sec: 83595.56738765333, Loss: 2.2844934463500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4400 , TFLOPS: 101.99302250120247, Tokens per sec: 83341.12421970577, Loss: 2.2836484909057617 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/optimizer.pt +[rank0]:[2024-08-27 17:40:57,028] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.023972694005351514, 'preprocessing_with_comm': 0.001790214009815827, 'state_converting': 3.33658050102531, : 3.3675697990111075}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400, took 15.99s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002400 + eval ppl=8.656322479248047, eval loss=2.158289909362793 +------------------------------------------------------------------ +iteration: 4401 , TFLOPS: 100.3592431689388, Tokens per sec: 82006.1210700916, Loss: 2.3060007095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4402 , TFLOPS: 100.07874558674233, Tokens per sec: 81776.91927502872, Loss: 2.333073616027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4403 , TFLOPS: 102.77155041214932, Tokens per sec: 83977.27941683178, Loss: 2.3113913536071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4404 , TFLOPS: 100.2765756955059, Tokens per sec: 81938.57134949949, Loss: 2.314565658569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4405 , TFLOPS: 102.23910914419834, Tokens per sec: 83542.20795053145, Loss: 2.2833032608032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4406 , TFLOPS: 102.7423546077876, Tokens per sec: 83953.42277352094, Loss: 2.2941441535949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4407 , TFLOPS: 101.4843296218529, Tokens per sec: 82925.45817306956, Loss: 2.2969462871551514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4408 , TFLOPS: 102.74322561416696, Tokens per sec: 83954.13449525522, Loss: 2.3037281036376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4409 , TFLOPS: 102.64597898981316, Tokens per sec: 83874.6717751449, Loss: 2.306335210800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4410 , TFLOPS: 103.33012882694308, Tokens per sec: 84433.70821864736, Loss: 2.3110082149505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4411 , TFLOPS: 102.04932002669857, Tokens per sec: 83387.12637701585, Loss: 2.312476873397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4412 , TFLOPS: 102.48323385919582, Tokens per sec: 83741.68853948511, Loss: 2.3261878490448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4413 , TFLOPS: 102.47986009213112, Tokens per sec: 83738.93174756806, Loss: 2.307413339614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4414 , TFLOPS: 103.28442891596852, Tokens per sec: 84396.36564496967, Loss: 2.3301751613616943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4415 , TFLOPS: 102.51194708177195, Tokens per sec: 83765.15085279619, Loss: 2.3171722888946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4416 , TFLOPS: 102.75992743713147, Tokens per sec: 83967.78198474269, Loss: 2.3221352100372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4417 , TFLOPS: 100.58551990579627, Tokens per sec: 82191.01761665924, Loss: 2.307713031768799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4418 , TFLOPS: 102.64657704443108, Tokens per sec: 83875.16046096852, Loss: 2.3176145553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4419 , TFLOPS: 103.26926650796, Tokens per sec: 84383.9760510708, Loss: 2.291809558868408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4420 , TFLOPS: 103.25011706118849, Tokens per sec: 84368.3285451632, Loss: 2.3210160732269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4421 , TFLOPS: 100.49767127438606, Tokens per sec: 82119.234238509, Loss: 2.31010103225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4422 , TFLOPS: 101.88712531320195, Tokens per sec: 83254.59290135441, Loss: 2.3126277923583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4423 , TFLOPS: 102.62329193583967, Tokens per sec: 83856.13359932645, Loss: 2.3074848651885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4424 , TFLOPS: 101.9656756961136, Tokens per sec: 83318.77844129827, Loss: 2.2991881370544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4425 , TFLOPS: 102.09284375345354, Tokens per sec: 83422.69073454775, Loss: 2.3231210708618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4426 , TFLOPS: 102.76615106399355, Tokens per sec: 83972.86747046211, Loss: 2.3158280849456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4427 , TFLOPS: 100.21524684342097, Tokens per sec: 81888.45796571579, Loss: 2.31676983833313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4428 , TFLOPS: 103.37513142584066, Tokens per sec: 84470.48100067652, Loss: 2.3091044425964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4429 , TFLOPS: 101.99814610008552, Tokens per sec: 83345.31084424627, Loss: 2.3075108528137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4430 , TFLOPS: 100.93428841946766, Tokens per sec: 82476.00534728038, Loss: 2.2943756580352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4431 , TFLOPS: 102.11361166594641, Tokens per sec: 83439.6607304593, Loss: 2.309098482131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4432 , TFLOPS: 100.82980673639246, Tokens per sec: 82390.63067444184, Loss: 2.3032989501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4433 , TFLOPS: 101.60654154160967, Tokens per sec: 83025.32067871751, Loss: 2.337904453277588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4434 , TFLOPS: 100.4914036950542, Tokens per sec: 82114.1128380951, Loss: 2.2920563220977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4435 , TFLOPS: 102.74127662003633, Tokens per sec: 83952.54192197951, Loss: 2.296116352081299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4436 , TFLOPS: 99.71537072770892, Tokens per sec: 81479.99632361149, Loss: 2.301362991333008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4437 , TFLOPS: 101.58206966965498, Tokens per sec: 83005.32408218132, Loss: 2.323861598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4438 , TFLOPS: 101.02759657423641, Tokens per sec: 82552.24984250752, Loss: 2.3178234100341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4439 , TFLOPS: 101.16645212445263, Tokens per sec: 82665.71228704906, Loss: 2.287332773208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4440 , TFLOPS: 100.31785655638396, Tokens per sec: 81972.30300358684, Loss: 2.2914865016937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4441 , TFLOPS: 102.7474300085811, Tokens per sec: 83957.57001416176, Loss: 2.3190577030181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4442 , TFLOPS: 100.81177022419678, Tokens per sec: 82375.89257602562, Loss: 2.3315486907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4443 , TFLOPS: 102.1452775361203, Tokens per sec: 83465.5357281304, Loss: 2.3216145038604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4444 , TFLOPS: 102.68785790754451, Tokens per sec: 83908.89211688246, Loss: 2.311861276626587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4445 , TFLOPS: 101.63508974868759, Tokens per sec: 83048.6481536171, Loss: 2.3115477561950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4446 , TFLOPS: 102.84353192271028, Tokens per sec: 84036.09736208005, Loss: 2.2857460975646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4447 , TFLOPS: 102.82666141817728, Tokens per sec: 84022.31203854072, Loss: 2.3044817447662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4448 , TFLOPS: 102.5680617492258, Tokens per sec: 83811.00359209278, Loss: 2.306656837463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4449 , TFLOPS: 101.99945364860102, Tokens per sec: 83346.37927579758, Loss: 2.2996585369110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4450 , TFLOPS: 102.75174646614111, Tokens per sec: 83961.09710274954, Loss: 2.3011205196380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4451 , TFLOPS: 103.5283102684889, Tokens per sec: 84595.6473761786, Loss: 2.319708824157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4452 , TFLOPS: 103.41483480979531, Tokens per sec: 84502.92365776189, Loss: 2.2949044704437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4453 , TFLOPS: 102.63620696738028, Tokens per sec: 83866.68680406077, Loss: 2.2990150451660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4454 , TFLOPS: 101.87770032877832, Tokens per sec: 83246.89150396117, Loss: 2.3052408695220947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4455 , TFLOPS: 101.3582309619854, Tokens per sec: 82822.41971202355, Loss: 2.297475576400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4456 , TFLOPS: 102.7316894455696, Tokens per sec: 83944.70799492692, Loss: 2.299328327178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4457 , TFLOPS: 103.29563739160069, Tokens per sec: 84405.52437894055, Loss: 2.3164713382720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4458 , TFLOPS: 103.37997375467762, Tokens per sec: 84474.43779222184, Loss: 2.3128631114959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4459 , TFLOPS: 100.82359378743753, Tokens per sec: 82385.55391391524, Loss: 2.290433645248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4460 , TFLOPS: 102.09386870807293, Tokens per sec: 83423.52825135193, Loss: 2.307131052017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4461 , TFLOPS: 102.63589380214349, Tokens per sec: 83866.4309086842, Loss: 2.336919069290161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4462 , TFLOPS: 102.92470076010032, Tokens per sec: 84102.42250858321, Loss: 2.307460308074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4463 , TFLOPS: 101.17251111543156, Tokens per sec: 82670.66324455023, Loss: 2.3030083179473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4464 , TFLOPS: 102.80801895590945, Tokens per sec: 84007.078802722, Loss: 2.286405563354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4465 , TFLOPS: 100.84229874740616, Tokens per sec: 82400.83821821412, Loss: 2.3170907497406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4466 , TFLOPS: 102.57397813606457, Tokens per sec: 83815.83802408006, Loss: 2.301043748855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4467 , TFLOPS: 102.0239850353415, Tokens per sec: 83366.4245033973, Loss: 2.3063876628875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4468 , TFLOPS: 101.54656209031965, Tokens per sec: 82976.30992505999, Loss: 2.3179972171783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4469 , TFLOPS: 101.20457609524118, Tokens per sec: 82696.86436497868, Loss: 2.306912422180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4470 , TFLOPS: 101.46728407104719, Tokens per sec: 82911.5298147147, Loss: 2.2905163764953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4471 , TFLOPS: 100.48209805948312, Tokens per sec: 82106.5089637216, Loss: 2.3065972328186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4472 , TFLOPS: 101.34886510869946, Tokens per sec: 82814.7666322051, Loss: 2.324683666229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4473 , TFLOPS: 101.87228033268195, Tokens per sec: 83242.46268562772, Loss: 2.3133318424224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4474 , TFLOPS: 100.34686649118187, Tokens per sec: 81996.00776809237, Loss: 2.3081295490264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4475 , TFLOPS: 101.51931335338158, Tokens per sec: 82954.04428066302, Loss: 2.316340208053589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4476 , TFLOPS: 100.51172842429412, Tokens per sec: 82130.72069756211, Loss: 2.294619560241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4477 , TFLOPS: 101.11703190193978, Tokens per sec: 82625.32975104409, Loss: 2.3000118732452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4478 , TFLOPS: 100.4989075540511, Tokens per sec: 82120.24443444797, Loss: 2.3206117153167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4479 , TFLOPS: 102.16522954124724, Tokens per sec: 83481.83902512994, Loss: 2.2928218841552734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4480 , TFLOPS: 100.24773338953925, Tokens per sec: 81915.00355882713, Loss: 2.29209041595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4481 , TFLOPS: 102.30835681527662, Tokens per sec: 83598.79200516315, Loss: 2.3109865188598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4482 , TFLOPS: 102.73765467501867, Tokens per sec: 83949.58233746843, Loss: 2.323007345199585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4483 , TFLOPS: 101.72211754819831, Tokens per sec: 83119.76080889218, Loss: 2.310990571975708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4484 , TFLOPS: 102.77703085871511, Tokens per sec: 83981.75763079993, Loss: 2.3162128925323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4485 , TFLOPS: 102.76705122399913, Tokens per sec: 83973.60301437485, Loss: 2.3112399578094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4486 , TFLOPS: 103.48108267329883, Tokens per sec: 84557.05649240235, Loss: 2.296889543533325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4487 , TFLOPS: 102.03808528438938, Tokens per sec: 83377.94617986704, Loss: 2.2769219875335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4488 , TFLOPS: 102.63622890018675, Tokens per sec: 83866.70472592155, Loss: 2.305863380432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4489 , TFLOPS: 103.43315573589585, Tokens per sec: 84517.89415810123, Loss: 2.294483184814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4490 , TFLOPS: 100.22619776900794, Tokens per sec: 81897.4062489148, Loss: 2.2889726161956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4491 , TFLOPS: 102.56157800208406, Tokens per sec: 83805.70555539677, Loss: 2.2844855785369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4492 , TFLOPS: 102.85305347657304, Tokens per sec: 84043.87766884793, Loss: 2.3207406997680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4493 , TFLOPS: 101.3957800390776, Tokens per sec: 82853.10202951482, Loss: 2.3073692321777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4494 , TFLOPS: 102.75287094067977, Tokens per sec: 83962.01593984163, Loss: 2.3002243041992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4495 , TFLOPS: 103.37243723601553, Tokens per sec: 84468.27950881577, Loss: 2.2677502632141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4496 , TFLOPS: 102.6571571082817, Tokens per sec: 83883.80570349592, Loss: 2.3201732635498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4497 , TFLOPS: 101.48123987602908, Tokens per sec: 82922.93346221978, Loss: 2.292656898498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4498 , TFLOPS: 102.11234489388801, Tokens per sec: 83438.62561840077, Loss: 2.2982676029205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4499 , TFLOPS: 102.75693558381518, Tokens per sec: 83965.3372643808, Loss: 2.3020803928375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4500 , TFLOPS: 102.13021460787412, Tokens per sec: 83453.22742170618, Loss: 2.306058883666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4501 , TFLOPS: 101.8757671366453, Tokens per sec: 83245.31184290448, Loss: 2.298912525177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4502 , TFLOPS: 102.85958329624275, Tokens per sec: 84049.21335259256, Loss: 2.3227691650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4503 , TFLOPS: 100.84030996853801, Tokens per sec: 82399.21313580523, Loss: 2.321169376373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4504 , TFLOPS: 102.59816693812093, Tokens per sec: 83835.60331691548, Loss: 2.29118013381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4505 , TFLOPS: 101.97967387995541, Tokens per sec: 83330.21671766072, Loss: 2.301147937774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4506 , TFLOPS: 101.56531427465666, Tokens per sec: 82991.63281760593, Loss: 2.307913064956665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4507 , TFLOPS: 101.3849989376671, Tokens per sec: 82844.29251402206, Loss: 2.317857265472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4508 , TFLOPS: 101.39491024666384, Tokens per sec: 82852.39129974309, Loss: 2.2804508209228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4509 , TFLOPS: 101.54024982106881, Tokens per sec: 82971.15200736288, Loss: 2.3250410556793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4510 , TFLOPS: 100.58670276948232, Tokens per sec: 82191.98416502672, Loss: 2.309493064880371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4511 , TFLOPS: 102.69123195275446, Tokens per sec: 83911.64913607916, Loss: 2.312917470932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4512 , TFLOPS: 100.21761568640778, Tokens per sec: 81890.39360829975, Loss: 2.3106698989868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4513 , TFLOPS: 101.46343517804411, Tokens per sec: 82908.38478516258, Loss: 2.3025729656219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4514 , TFLOPS: 101.06355874737937, Tokens per sec: 82581.63545993132, Loss: 2.301298141479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4515 , TFLOPS: 101.1945889451754, Tokens per sec: 82688.70360756782, Loss: 2.2769863605499268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4516 , TFLOPS: 100.32512713233261, Tokens per sec: 81978.24397835576, Loss: 2.307023763656616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4517 , TFLOPS: 101.91758092864148, Tokens per sec: 83279.47896872734, Loss: 2.3113722801208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4518 , TFLOPS: 100.9538923962605, Tokens per sec: 82492.02426136906, Loss: 2.299327850341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4519 , TFLOPS: 101.64949360249592, Tokens per sec: 83060.41791335208, Loss: 2.315646171569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4520 , TFLOPS: 102.20907145119074, Tokens per sec: 83517.66337833593, Loss: 2.289644718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4521 , TFLOPS: 101.6735888837692, Tokens per sec: 83080.1067879482, Loss: 2.3099405765533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4522 , TFLOPS: 102.19431257545072, Tokens per sec: 83505.6035210415, Loss: 2.31793212890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4523 , TFLOPS: 102.91294475052881, Tokens per sec: 84092.81637053573, Loss: 2.338852643966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4524 , TFLOPS: 103.39419610669654, Tokens per sec: 84486.05924217241, Loss: 2.3271632194519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4525 , TFLOPS: 101.88362418122486, Tokens per sec: 83251.73203628878, Loss: 2.2868733406066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4526 , TFLOPS: 102.70799876268336, Tokens per sec: 83925.34972808814, Loss: 2.3010241985321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4527 , TFLOPS: 103.26921892490444, Tokens per sec: 84383.9371697312, Loss: 2.308973550796509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4528 , TFLOPS: 103.21235163756243, Tokens per sec: 84337.46944534985, Loss: 2.3261983394622803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4529 , TFLOPS: 101.98320729997383, Tokens per sec: 83333.10396611609, Loss: 2.298029661178589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4530 , TFLOPS: 102.83357697606948, Tokens per sec: 84027.96291891675, Loss: 2.3188695907592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4531 , TFLOPS: 101.2724093226627, Tokens per sec: 82752.29264128744, Loss: 2.316141366958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4532 , TFLOPS: 102.77786082145147, Tokens per sec: 83982.43581471694, Loss: 2.290471315383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4533 , TFLOPS: 102.5964627590952, Tokens per sec: 83834.2107883641, Loss: 2.287632942199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4534 , TFLOPS: 103.44255488835685, Tokens per sec: 84525.57444753282, Loss: 2.2998783588409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4535 , TFLOPS: 101.55926662194581, Tokens per sec: 82986.69112489547, Loss: 2.3259942531585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4536 , TFLOPS: 101.58716143996989, Tokens per sec: 83009.48469877944, Loss: 2.3141121864318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4537 , TFLOPS: 101.82044978326068, Tokens per sec: 83200.11060945857, Loss: 2.297119379043579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4538 , TFLOPS: 102.81714715318938, Tokens per sec: 84014.53768770014, Loss: 2.310323476791382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4539 , TFLOPS: 101.52779378183115, Tokens per sec: 82960.9738570548, Loss: 2.293771743774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4540 , TFLOPS: 102.71513569476323, Tokens per sec: 83931.18149901158, Loss: 2.3060555458068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4541 , TFLOPS: 100.87516773945633, Tokens per sec: 82427.69631774143, Loss: 2.3153910636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4542 , TFLOPS: 101.87827447374882, Tokens per sec: 83247.36065259637, Loss: 2.313682794570923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4543 , TFLOPS: 102.06278124502741, Tokens per sec: 83398.12588503494, Loss: 2.30871844291687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4544 , TFLOPS: 100.9495829345302, Tokens per sec: 82488.50288925382, Loss: 2.3005173206329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4545 , TFLOPS: 102.05454286614243, Tokens per sec: 83391.3940935733, Loss: 2.3056881427764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4546 , TFLOPS: 100.22070847928714, Tokens per sec: 81892.92080897704, Loss: 2.3203063011169434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4547 , TFLOPS: 101.54197940389614, Tokens per sec: 82972.5652940145, Loss: 2.312868595123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4548 , TFLOPS: 100.88461803560119, Tokens per sec: 82435.41839799366, Loss: 2.289454221725464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4549 , TFLOPS: 101.96946489936335, Tokens per sec: 83321.8746968164, Loss: 2.301774740219116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4550 , TFLOPS: 99.66176729195111, Tokens per sec: 81436.1955763786, Loss: 2.3186986446380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4551 , TFLOPS: 101.6866271775896, Tokens per sec: 83090.76071346438, Loss: 2.3080809116363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4552 , TFLOPS: 100.33407058831736, Tokens per sec: 81985.55190646595, Loss: 2.302006721496582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4553 , TFLOPS: 101.35070707471355, Tokens per sec: 82816.27175005163, Loss: 2.2986433506011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4554 , TFLOPS: 100.05758070354827, Tokens per sec: 81759.6249041383, Loss: 2.2960586547851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4555 , TFLOPS: 102.68964658891096, Tokens per sec: 83910.35369447173, Loss: 2.2975096702575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4556 , TFLOPS: 100.29764437730277, Tokens per sec: 81955.78711175184, Loss: 2.3120322227478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4557 , TFLOPS: 102.24243309950877, Tokens per sec: 83544.92403998182, Loss: 2.3075718879699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4558 , TFLOPS: 102.01678946431718, Tokens per sec: 83360.54481708286, Loss: 2.3263437747955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4559 , TFLOPS: 101.78448224387726, Tokens per sec: 83170.72060714154, Loss: 2.287998676300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4560 , TFLOPS: 102.28877118946147, Tokens per sec: 83582.78808613072, Loss: 2.3094749450683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4561 , TFLOPS: 102.22904519231712, Tokens per sec: 83533.98443638014, Loss: 2.3040709495544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4562 , TFLOPS: 102.78350932701564, Tokens per sec: 83987.0513540189, Loss: 2.305616855621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4563 , TFLOPS: 101.47072330750126, Tokens per sec: 82914.34010335515, Loss: 2.308202028274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4564 , TFLOPS: 102.73874402556508, Tokens per sec: 83950.47247384228, Loss: 2.3179030418395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4565 , TFLOPS: 103.40598118835356, Tokens per sec: 84495.68913576935, Loss: 2.307023525238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4566 , TFLOPS: 103.30621599975817, Tokens per sec: 84414.16843198363, Loss: 2.303044080734253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4567 , TFLOPS: 102.02363208294629, Tokens per sec: 83366.13609690938, Loss: 2.3079376220703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4568 , TFLOPS: 102.78719579964668, Tokens per sec: 83990.06366570397, Loss: 2.2946536540985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4569 , TFLOPS: 101.40227943940626, Tokens per sec: 82858.4128568328, Loss: 2.3051609992980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4570 , TFLOPS: 102.75527619638181, Tokens per sec: 83963.98133619255, Loss: 2.3032312393188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4571 , TFLOPS: 102.61458394626328, Tokens per sec: 83849.01807687999, Loss: 2.3100690841674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4572 , TFLOPS: 102.67608891217662, Tokens per sec: 83899.27536780646, Loss: 2.2843966484069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4573 , TFLOPS: 101.3315815231273, Tokens per sec: 82800.643769515, Loss: 2.3023505210876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4574 , TFLOPS: 101.44517386289239, Tokens per sec: 82893.46299446402, Loss: 2.3096225261688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4575 , TFLOPS: 102.74062029226327, Tokens per sec: 83952.00561966076, Loss: 2.320070266723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4576 , TFLOPS: 102.76048785999534, Tokens per sec: 83968.23992069133, Loss: 2.313628911972046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4577 , TFLOPS: 100.89056784152243, Tokens per sec: 82440.28013757354, Loss: 2.3275885581970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4578 , TFLOPS: 102.84043432019365, Tokens per sec: 84033.56623132434, Loss: 2.3091611862182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4579 , TFLOPS: 101.0102793775869, Tokens per sec: 82538.0995153409, Loss: 2.313023090362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4580 , TFLOPS: 102.63862354759341, Tokens per sec: 83868.66145396123, Loss: 2.3281030654907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4581 , TFLOPS: 101.50194688488558, Tokens per sec: 82939.85369220226, Loss: 2.3243460655212402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4582 , TFLOPS: 101.03716071984523, Tokens per sec: 82560.0649520875, Loss: 2.31080961227417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4583 , TFLOPS: 102.11894804516747, Tokens per sec: 83444.02122329145, Loss: 2.2936995029449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4584 , TFLOPS: 99.62886361391999, Tokens per sec: 81409.30913404333, Loss: 2.317106008529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4585 , TFLOPS: 101.48640552949625, Tokens per sec: 82927.15445064356, Loss: 2.295501470565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4586 , TFLOPS: 100.73297607074039, Tokens per sec: 82311.50784489446, Loss: 2.3129196166992188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4587 , TFLOPS: 101.90566713911514, Tokens per sec: 83269.74390461713, Loss: 2.309680938720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4588 , TFLOPS: 99.86508057491298, Tokens per sec: 81602.32809363624, Loss: 2.312285900115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4589 , TFLOPS: 101.05791692239436, Tokens per sec: 82577.02537950258, Loss: 2.3148553371429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4590 , TFLOPS: 101.04852773627081, Tokens per sec: 82569.35323382186, Loss: 2.2803735733032227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4591 , TFLOPS: 101.31215542155485, Tokens per sec: 82784.77020184782, Loss: 2.308669328689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4592 , TFLOPS: 99.81800359760072, Tokens per sec: 81563.86028360519, Loss: 2.3099892139434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4593 , TFLOPS: 102.60669735061694, Tokens per sec: 83842.57373655833, Loss: 2.3028011322021484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4594 , TFLOPS: 100.85456989918437, Tokens per sec: 82410.86529221962, Loss: 2.2960193157196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4595 , TFLOPS: 101.44496702536797, Tokens per sec: 82893.29398219836, Loss: 2.3162333965301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4596 , TFLOPS: 102.70115156611782, Tokens per sec: 83919.75470751192, Loss: 2.3075571060180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4597 , TFLOPS: 101.25974218019502, Tokens per sec: 82741.94199309582, Loss: 2.3074588775634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4598 , TFLOPS: 102.81645125458509, Tokens per sec: 84013.96905103655, Loss: 2.312349796295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4599 , TFLOPS: 102.0692053088411, Tokens per sec: 83403.37515294724, Loss: 2.3265910148620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4600 , TFLOPS: 103.38475374307323, Tokens per sec: 84478.34364378774, Loss: 2.3101747035980225 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/optimizer.pt +[rank0]:[2024-08-27 18:33:49,371] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007687582983635366, 'preprocessing_with_comm': 0.0016850910033099353, 'state_converting': 3.322918573016068, : 3.333903416991234}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600, took 15.96s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002600 + eval ppl=8.028962135314941, eval loss=2.083055257797241 +------------------------------------------------------------------ +iteration: 4601 , TFLOPS: 100.55292431124445, Tokens per sec: 82164.3829172657, Loss: 2.305417776107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4602 , TFLOPS: 100.31449842182975, Tokens per sec: 81969.55898539654, Loss: 2.303450345993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4603 , TFLOPS: 100.6890397205157, Tokens per sec: 82275.60632210359, Loss: 2.313088893890381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4604 , TFLOPS: 100.0575960111024, Tokens per sec: 81759.63741233484, Loss: 2.2908935546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4605 , TFLOPS: 100.57770047381469, Tokens per sec: 82184.62815749699, Loss: 2.321455240249634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4606 , TFLOPS: 102.20485421042488, Tokens per sec: 83514.2173623447, Loss: 2.287771224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4607 , TFLOPS: 101.14237694378535, Tokens per sec: 82646.03983717553, Loss: 2.289451837539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4608 , TFLOPS: 102.24500636650173, Tokens per sec: 83547.02672268356, Loss: 2.300848960876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4609 , TFLOPS: 101.97240197924295, Tokens per sec: 83324.27466039319, Loss: 2.3114373683929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4610 , TFLOPS: 103.195140548478, Tokens per sec: 84323.40582140611, Loss: 2.2918179035186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4611 , TFLOPS: 102.59144873711195, Tokens per sec: 83830.11370192916, Loss: 2.309023380279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4612 , TFLOPS: 103.33613912528533, Tokens per sec: 84438.61938814164, Loss: 2.2869560718536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4613 , TFLOPS: 101.86165318143748, Tokens per sec: 83233.77896677866, Loss: 2.3003273010253906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4614 , TFLOPS: 100.56657647239516, Tokens per sec: 82175.53844957982, Loss: 2.2985970973968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4615 , TFLOPS: 103.38405535210613, Tokens per sec: 84477.77297055033, Loss: 2.310225486755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4616 , TFLOPS: 102.57387850022002, Tokens per sec: 83815.75660906662, Loss: 2.3008639812469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4617 , TFLOPS: 101.96476243423183, Tokens per sec: 83318.03219150522, Loss: 2.3248016834259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4618 , TFLOPS: 102.63674786827576, Tokens per sec: 83867.12878810649, Loss: 2.28027081489563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4619 , TFLOPS: 102.01796646992722, Tokens per sec: 83361.50657866565, Loss: 2.312225341796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4620 , TFLOPS: 102.79084151983854, Tokens per sec: 83993.04267752192, Loss: 2.315612554550171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4621 , TFLOPS: 102.82254523733536, Tokens per sec: 84018.94860121487, Loss: 2.296356439590454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4622 , TFLOPS: 103.35009275552723, Tokens per sec: 84450.0212586107, Loss: 2.3052592277526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4623 , TFLOPS: 100.53401909317247, Tokens per sec: 82148.93497691547, Loss: 2.2988271713256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4624 , TFLOPS: 100.33935239910599, Tokens per sec: 81989.86781002724, Loss: 2.3127219676971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4625 , TFLOPS: 103.33216990577772, Tokens per sec: 84435.37603670519, Loss: 2.3118672370910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4626 , TFLOPS: 102.76467942800039, Tokens per sec: 83971.66495880882, Loss: 2.306239128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4627 , TFLOPS: 99.93765431896055, Tokens per sec: 81661.6299681119, Loss: 2.30367374420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4628 , TFLOPS: 102.64900715907876, Tokens per sec: 83877.14617020385, Loss: 2.326436996459961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4629 , TFLOPS: 102.11511153133266, Tokens per sec: 83440.88630907638, Loss: 2.3105621337890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4630 , TFLOPS: 100.24761721641764, Tokens per sec: 81914.90863077891, Loss: 2.3117215633392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4631 , TFLOPS: 103.31441769077149, Tokens per sec: 84420.87024484092, Loss: 2.3234260082244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4632 , TFLOPS: 101.38504327648768, Tokens per sec: 82844.32874441372, Loss: 2.3118176460266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4633 , TFLOPS: 100.37750294266556, Tokens per sec: 82021.04160125212, Loss: 2.315931558609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4634 , TFLOPS: 102.20110684078419, Tokens per sec: 83511.15529014543, Loss: 2.328033685684204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4635 , TFLOPS: 100.28189386307122, Tokens per sec: 81942.91696111893, Loss: 2.290337085723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4636 , TFLOPS: 101.41328976867321, Tokens per sec: 82867.40968030793, Loss: 2.3190855979919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4637 , TFLOPS: 100.90087213732112, Tokens per sec: 82448.70004292681, Loss: 2.305847406387329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4638 , TFLOPS: 100.94610506693628, Tokens per sec: 82485.66103411457, Loss: 2.291123867034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4639 , TFLOPS: 100.39485241747217, Tokens per sec: 82035.21830373182, Loss: 2.289642333984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4640 , TFLOPS: 99.87628062120099, Tokens per sec: 81611.47993977301, Loss: 2.3139429092407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4641 , TFLOPS: 102.34994915213485, Tokens per sec: 83632.778174292, Loss: 2.307314872741699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4642 , TFLOPS: 99.43558686934736, Tokens per sec: 81251.37773066701, Loss: 2.28043532371521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4643 , TFLOPS: 101.48449391319917, Tokens per sec: 82925.59241975784, Loss: 2.276122570037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4644 , TFLOPS: 101.39733917001855, Tokens per sec: 82854.37603554227, Loss: 2.308767080307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4645 , TFLOPS: 102.34402861077956, Tokens per sec: 83627.94034754232, Loss: 2.290067195892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4646 , TFLOPS: 102.2961588279361, Tokens per sec: 83588.82471570307, Loss: 2.3076400756835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4647 , TFLOPS: 102.84372347589327, Tokens per sec: 84036.25388511691, Loss: 2.3099112510681152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4648 , TFLOPS: 102.04163895954737, Tokens per sec: 83380.84997931836, Loss: 2.3189663887023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4649 , TFLOPS: 102.691247354393, Tokens per sec: 83911.66172115447, Loss: 2.3213536739349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4650 , TFLOPS: 103.39828599866526, Tokens per sec: 84489.40119818323, Loss: 2.320836067199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4651 , TFLOPS: 102.7624264877534, Tokens per sec: 83969.82402333716, Loss: 2.2889838218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4652 , TFLOPS: 101.44137353458702, Tokens per sec: 82890.35764837655, Loss: 2.287196397781372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4653 , TFLOPS: 102.87767272483872, Tokens per sec: 84063.99469036164, Loss: 2.299755096435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4654 , TFLOPS: 103.34318132576333, Tokens per sec: 84444.37375143458, Loss: 2.3053083419799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4655 , TFLOPS: 102.01105425850915, Tokens per sec: 83355.85843277979, Loss: 2.300654888153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4656 , TFLOPS: 102.12365489581872, Tokens per sec: 83447.86731212371, Loss: 2.291351318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4657 , TFLOPS: 102.12478889320701, Tokens per sec: 83448.79393058136, Loss: 2.3050436973571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4658 , TFLOPS: 101.88931411572048, Tokens per sec: 83256.38142823709, Loss: 2.2941370010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4659 , TFLOPS: 102.77111010201143, Tokens per sec: 83976.91962808352, Loss: 2.290560007095337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4660 , TFLOPS: 103.30496244183237, Tokens per sec: 84413.14411753301, Loss: 2.286221981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4661 , TFLOPS: 101.436035512949, Tokens per sec: 82885.99581348323, Loss: 2.3263132572174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4662 , TFLOPS: 100.88362204096887, Tokens per sec: 82434.6045451404, Loss: 2.298689842224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4663 , TFLOPS: 103.50034846027837, Tokens per sec: 84572.7990628888, Loss: 2.2881200313568115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4664 , TFLOPS: 102.14660463734735, Tokens per sec: 83466.62013670565, Loss: 2.299391984939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4665 , TFLOPS: 99.70236256983515, Tokens per sec: 81469.36702295307, Loss: 2.2965087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4666 , TFLOPS: 102.75913414526765, Tokens per sec: 83967.13376553962, Loss: 2.293700933456421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4667 , TFLOPS: 102.75881244267562, Tokens per sec: 83966.8708940703, Loss: 2.292168378829956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4668 , TFLOPS: 99.70220369487058, Tokens per sec: 81469.2372021297, Loss: 2.3074164390563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4669 , TFLOPS: 102.00563516372694, Tokens per sec: 83351.43035093333, Loss: 2.3254427909851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4670 , TFLOPS: 102.706761783148, Tokens per sec: 83924.33896026711, Loss: 2.2962136268615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4671 , TFLOPS: 99.93989108115608, Tokens per sec: 81663.45768408124, Loss: 2.324949026107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4672 , TFLOPS: 102.19056393653425, Tokens per sec: 83502.5404116843, Loss: 2.321312427520752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4673 , TFLOPS: 100.99638000663346, Tokens per sec: 82526.74198153321, Loss: 2.2842636108398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4674 , TFLOPS: 101.52389570806554, Tokens per sec: 82957.78864063558, Loss: 2.3040661811828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4675 , TFLOPS: 101.48195764864386, Tokens per sec: 82923.51997270042, Loss: 2.316033124923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4676 , TFLOPS: 100.81346064034044, Tokens per sec: 82377.27385857179, Loss: 2.307063579559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4677 , TFLOPS: 100.96373065070244, Tokens per sec: 82500.06335233279, Loss: 2.3093464374542236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4678 , TFLOPS: 101.0923292089525, Tokens per sec: 82605.1445447003, Loss: 2.286003589630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4679 , TFLOPS: 101.51730371841282, Tokens per sec: 82952.40215619742, Loss: 2.3188729286193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4680 , TFLOPS: 100.18058933487848, Tokens per sec: 81860.13842332209, Loss: 2.295415163040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4681 , TFLOPS: 100.31905035114156, Tokens per sec: 81973.2784840143, Loss: 2.2841732501983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4682 , TFLOPS: 102.20459694716432, Tokens per sec: 83514.00714591196, Loss: 2.3038136959075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4683 , TFLOPS: 102.42767605905266, Tokens per sec: 83696.29083080313, Loss: 2.2827131748199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4684 , TFLOPS: 102.14738664805142, Tokens per sec: 83467.25913778269, Loss: 2.293647050857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4685 , TFLOPS: 102.7097175096879, Tokens per sec: 83926.75416051004, Loss: 2.259549617767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4686 , TFLOPS: 101.99953269368255, Tokens per sec: 83346.44386556851, Loss: 2.2971911430358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4687 , TFLOPS: 102.53550898752698, Tokens per sec: 83784.40389252602, Loss: 2.296515464782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4688 , TFLOPS: 103.35601251864469, Tokens per sec: 84454.85844944222, Loss: 2.2853150367736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4689 , TFLOPS: 102.20865637707175, Tokens per sec: 83517.32421058834, Loss: 2.301697015762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4690 , TFLOPS: 101.27091664056866, Tokens per sec: 82751.07293232334, Loss: 2.3100972175598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4691 , TFLOPS: 102.0492996217748, Tokens per sec: 83387.10970362739, Loss: 2.3040928840637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4692 , TFLOPS: 103.3644253980566, Tokens per sec: 84461.73282977648, Loss: 2.320796489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4693 , TFLOPS: 102.08315268196807, Tokens per sec: 83414.7719105666, Loss: 2.297760248184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4694 , TFLOPS: 102.1605129099996, Tokens per sec: 83477.98494432059, Loss: 2.3073413372039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4695 , TFLOPS: 102.08164725850428, Tokens per sec: 83413.5417903013, Loss: 2.2997615337371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4696 , TFLOPS: 102.80744265019781, Tokens per sec: 84006.60788848961, Loss: 2.300572633743286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4697 , TFLOPS: 102.85586100428736, Tokens per sec: 84046.1717720155, Loss: 2.295302629470825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4698 , TFLOPS: 103.33035773387513, Tokens per sec: 84433.89526439412, Loss: 2.284902811050415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4699 , TFLOPS: 101.3008124566792, Tokens per sec: 82775.50157325402, Loss: 2.301079511642456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4700 , TFLOPS: 99.95476449637648, Tokens per sec: 81675.61113453361, Loss: 2.298750638961792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4701 , TFLOPS: 103.29484911078515, Tokens per sec: 84404.88025439401, Loss: 2.312595844268799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4702 , TFLOPS: 102.2663757756099, Tokens per sec: 83564.48821696336, Loss: 2.30391788482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4703 , TFLOPS: 100.39825398379664, Tokens per sec: 82037.99781113965, Loss: 2.3130922317504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4704 , TFLOPS: 102.82685890767993, Tokens per sec: 84022.47341229717, Loss: 2.311140775680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4705 , TFLOPS: 102.65737938058122, Tokens per sec: 83883.98732791367, Loss: 2.3202576637268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4706 , TFLOPS: 98.90605913340056, Tokens per sec: 80818.6869863683, Loss: 2.288583517074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4707 , TFLOPS: 102.74855499354263, Tokens per sec: 83958.4892683335, Loss: 2.3080503940582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4708 , TFLOPS: 102.64026104059411, Tokens per sec: 83869.99949164492, Loss: 2.3099677562713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4709 , TFLOPS: 99.84066469057025, Tokens per sec: 81582.37724601902, Loss: 2.325481414794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4710 , TFLOPS: 102.23923529106682, Tokens per sec: 83542.31102838498, Loss: 2.3047351837158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4711 , TFLOPS: 100.98852005606771, Tokens per sec: 82520.3194135928, Loss: 2.304706573486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4712 , TFLOPS: 101.59043673604634, Tokens per sec: 83012.16102751714, Loss: 2.300046443939209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4713 , TFLOPS: 101.41373035809863, Tokens per sec: 82867.76969726922, Loss: 2.309536933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4714 , TFLOPS: 101.58575477995448, Tokens per sec: 83008.33528067022, Loss: 2.3034231662750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4715 , TFLOPS: 100.35119255223229, Tokens per sec: 81999.5427039394, Loss: 2.3293471336364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4716 , TFLOPS: 101.0164374399023, Tokens per sec: 82543.13142658144, Loss: 2.3040802478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4717 , TFLOPS: 102.29864112732263, Tokens per sec: 83590.85306643188, Loss: 2.308054208755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4718 , TFLOPS: 98.75076414321042, Tokens per sec: 80691.79145223524, Loss: 2.28714656829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4719 , TFLOPS: 100.92423998531699, Tokens per sec: 82467.79451306615, Loss: 2.287741184234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4720 , TFLOPS: 102.29646065570493, Tokens per sec: 83589.07134694273, Loss: 2.2776734828948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4721 , TFLOPS: 101.69363802051332, Tokens per sec: 83096.48945369251, Loss: 2.2810826301574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4722 , TFLOPS: 102.91626604069565, Tokens per sec: 84095.53028222866, Loss: 2.2888317108154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4723 , TFLOPS: 101.56919759075207, Tokens per sec: 82994.80597515295, Loss: 2.3287758827209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4724 , TFLOPS: 103.45834352205922, Tokens per sec: 84538.47574656662, Loss: 2.280404567718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4725 , TFLOPS: 102.66058307169118, Tokens per sec: 83886.60514638996, Loss: 2.2952399253845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4726 , TFLOPS: 102.7800479161101, Tokens per sec: 83984.22294606334, Loss: 2.3056633472442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4727 , TFLOPS: 102.03122467332884, Tokens per sec: 83372.34019796162, Loss: 2.3138339519500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4728 , TFLOPS: 101.34191560792875, Tokens per sec: 82809.08801622927, Loss: 2.286550283432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4729 , TFLOPS: 102.02792077726231, Tokens per sec: 83369.64049943582, Loss: 2.2915377616882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4730 , TFLOPS: 103.3007191614716, Tokens per sec: 84409.67682391847, Loss: 2.3082058429718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4731 , TFLOPS: 101.33330442664436, Tokens per sec: 82802.05159833028, Loss: 2.3105709552764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4732 , TFLOPS: 101.29466165356752, Tokens per sec: 82770.47559369571, Loss: 2.3070497512817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4733 , TFLOPS: 101.9072058348247, Tokens per sec: 83271.00121249096, Loss: 2.2713429927825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4734 , TFLOPS: 102.23997887157492, Tokens per sec: 83542.91862715961, Loss: 2.2957146167755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4735 , TFLOPS: 103.38300390926902, Tokens per sec: 84476.91380954158, Loss: 2.2880072593688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4736 , TFLOPS: 102.5569328296233, Tokens per sec: 83801.9098654017, Loss: 2.3155529499053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4737 , TFLOPS: 101.35825057323747, Tokens per sec: 82822.43573688246, Loss: 2.310375213623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4738 , TFLOPS: 100.89010913508008, Tokens per sec: 82439.90531673134, Loss: 2.3029327392578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4739 , TFLOPS: 102.71528749566261, Tokens per sec: 83931.30553943416, Loss: 2.3073787689208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4740 , TFLOPS: 102.82909623457402, Tokens per sec: 84024.30158969625, Loss: 2.3182485103607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4741 , TFLOPS: 97.18819848951163, Tokens per sec: 79414.9788325796, Loss: 2.30845046043396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4742 , TFLOPS: 102.73605425045143, Tokens per sec: 83948.27458935593, Loss: 2.311920642852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4743 , TFLOPS: 102.6112524453424, Tokens per sec: 83846.29582171708, Loss: 2.314746856689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4744 , TFLOPS: 99.97842173636604, Tokens per sec: 81694.9420743206, Loss: 2.3053929805755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4745 , TFLOPS: 103.37172137512691, Tokens per sec: 84467.6945604558, Loss: 2.3035061359405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4746 , TFLOPS: 102.754893749399, Tokens per sec: 83963.66882891784, Loss: 2.2798616886138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4747 , TFLOPS: 100.14706059453762, Tokens per sec: 81832.741226483, Loss: 2.3085505962371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4748 , TFLOPS: 101.84959672326741, Tokens per sec: 83223.92732444705, Loss: 2.303302526473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4749 , TFLOPS: 101.75692239270037, Tokens per sec: 83148.20074329122, Loss: 2.3095569610595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4750 , TFLOPS: 101.50387399450621, Tokens per sec: 82941.4283830815, Loss: 2.3044912815093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4751 , TFLOPS: 100.69374210536131, Tokens per sec: 82279.44876181109, Loss: 2.2983815670013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4752 , TFLOPS: 100.82845078173551, Tokens per sec: 82389.52268898628, Loss: 2.2780871391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4753 , TFLOPS: 100.79869124542056, Tokens per sec: 82365.20540578471, Loss: 2.3167026042938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4754 , TFLOPS: 101.19695223607324, Tokens per sec: 82690.63471339697, Loss: 2.2950327396392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4755 , TFLOPS: 101.65538526846471, Tokens per sec: 83065.23214528016, Loss: 2.3044724464416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4756 , TFLOPS: 99.43796276981534, Tokens per sec: 81253.31914010052, Loss: 2.294769763946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4757 , TFLOPS: 101.7100040373857, Tokens per sec: 83109.86254737768, Loss: 2.3073902130126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4758 , TFLOPS: 101.71789344466217, Tokens per sec: 83116.3091851547, Loss: 2.3071844577789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4759 , TFLOPS: 101.75782892978405, Tokens per sec: 83148.94149807864, Loss: 2.3056817054748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4760 , TFLOPS: 102.820587965206, Tokens per sec: 84017.3492637733, Loss: 2.2935516834259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4761 , TFLOPS: 102.00840081763917, Tokens per sec: 83353.69023793927, Loss: 2.32464599609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4762 , TFLOPS: 102.58034135649103, Tokens per sec: 83821.0375752944, Loss: 2.29679012298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4763 , TFLOPS: 103.34398413323053, Tokens per sec: 84445.02974608215, Loss: 2.296062469482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4764 , TFLOPS: 102.13345432503996, Tokens per sec: 83455.87467800012, Loss: 2.293257236480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4765 , TFLOPS: 102.71901849932397, Tokens per sec: 83934.35423857043, Loss: 2.317732810974121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4766 , TFLOPS: 100.83406774215064, Tokens per sec: 82394.1124519349, Loss: 2.30849552154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4767 , TFLOPS: 102.46592257459614, Tokens per sec: 83727.5430432066, Loss: 2.316955804824829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4768 , TFLOPS: 102.57762134216296, Tokens per sec: 83818.81498156753, Loss: 2.3085176944732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4769 , TFLOPS: 101.21524881318273, Tokens per sec: 82705.58531754892, Loss: 2.299840211868286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4770 , TFLOPS: 101.37669152183236, Tokens per sec: 82837.50431069158, Loss: 2.3201584815979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4771 , TFLOPS: 101.98382940352386, Tokens per sec: 83333.61230294118, Loss: 2.30622935295105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4772 , TFLOPS: 101.66214905661518, Tokens per sec: 83070.75901069329, Loss: 2.3006386756896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4773 , TFLOPS: 102.83009004513187, Tokens per sec: 84025.11365788603, Loss: 2.293652057647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4774 , TFLOPS: 102.60946067771643, Tokens per sec: 83844.83172226569, Loss: 2.298300266265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4775 , TFLOPS: 100.75130818477875, Tokens per sec: 82326.4874871858, Loss: 2.3069663047790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4776 , TFLOPS: 101.58299040180404, Tokens per sec: 83006.07643612206, Loss: 2.3197731971740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4777 , TFLOPS: 103.3370198559058, Tokens per sec: 84439.33905580365, Loss: 2.2926247119903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4778 , TFLOPS: 102.69431983128835, Tokens per sec: 83914.17232111825, Loss: 2.274876594543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4779 , TFLOPS: 100.4502579086297, Tokens per sec: 82080.49155682085, Loss: 2.296137571334839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4780 , TFLOPS: 103.37306289335575, Tokens per sec: 84468.79074953432, Loss: 2.3089890480041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4781 , TFLOPS: 102.69769236993152, Tokens per sec: 83916.92810926045, Loss: 2.3179574012756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4782 , TFLOPS: 100.42333627455258, Tokens per sec: 82058.49319659194, Loss: 2.301405429840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4783 , TFLOPS: 102.01229315755212, Tokens per sec: 83356.87076907989, Loss: 2.2900145053863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4784 , TFLOPS: 102.71709801575894, Tokens per sec: 83932.7849620118, Loss: 2.297544002532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4785 , TFLOPS: 100.24517618821882, Tokens per sec: 81912.91400379999, Loss: 2.310153007507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4786 , TFLOPS: 102.72998895204896, Tokens per sec: 83943.31847789661, Loss: 2.298151969909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4787 , TFLOPS: 101.06157363588211, Tokens per sec: 82580.0133742256, Loss: 2.3030762672424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4788 , TFLOPS: 101.50007042849896, Tokens per sec: 82938.32039138446, Loss: 2.2815237045288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4789 , TFLOPS: 100.93392795424552, Tokens per sec: 82475.71080186818, Loss: 2.3062655925750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4790 , TFLOPS: 101.65170386972626, Tokens per sec: 83062.22397960332, Loss: 2.304508924484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4791 , TFLOPS: 100.21360934757405, Tokens per sec: 81887.11992570637, Loss: 2.3146731853485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4792 , TFLOPS: 100.99309030493471, Tokens per sec: 82524.0538815907, Loss: 2.2763407230377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4793 , TFLOPS: 101.66976343992013, Tokens per sec: 83076.98091930345, Loss: 2.3069934844970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4794 , TFLOPS: 99.2871463855056, Tokens per sec: 81130.0831901219, Loss: 2.2983129024505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4795 , TFLOPS: 101.71153605686283, Tokens per sec: 83111.11439993014, Loss: 2.304180383682251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4796 , TFLOPS: 101.64353373827255, Tokens per sec: 83055.54795487461, Loss: 2.303154945373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4797 , TFLOPS: 101.81141400329663, Tokens per sec: 83192.72724104827, Loss: 2.3051068782806396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4798 , TFLOPS: 102.6824950677299, Tokens per sec: 83904.51000241816, Loss: 2.295191526412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4799 , TFLOPS: 102.06584321345613, Tokens per sec: 83400.62789826014, Loss: 2.306042432785034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4800 , TFLOPS: 102.60651065290799, Tokens per sec: 83842.42118105432, Loss: 2.2904579639434814 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/model.pt +[rank0]:[2024-08-27 19:26:45,754] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007789055001921952, 'preprocessing_with_comm': 0.0015716630150564015, 'state_converting': 3.3272144490038045, : 3.338156811980298}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800, took 15.71s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0002800 + eval ppl=7.855510234832764, eval loss=2.0612151622772217 +------------------------------------------------------------------ +iteration: 4801 , TFLOPS: 101.76452563832899, Tokens per sec: 83154.41355101932, Loss: 2.284130334854126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4802 , TFLOPS: 102.58700079659741, Tokens per sec: 83826.47917523455, Loss: 2.292233467102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4803 , TFLOPS: 102.01646935313848, Tokens per sec: 83360.28324599835, Loss: 2.3067872524261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4804 , TFLOPS: 102.10050256900115, Tokens per sec: 83428.94894988716, Loss: 2.3049192428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4805 , TFLOPS: 100.90924361078129, Tokens per sec: 82455.54058939201, Loss: 2.3233346939086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4806 , TFLOPS: 101.14679559083035, Tokens per sec: 82649.6504274221, Loss: 2.2972545623779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4807 , TFLOPS: 102.76234822892665, Tokens per sec: 83969.76007603527, Loss: 2.31183123588562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4808 , TFLOPS: 102.00574596616428, Tokens per sec: 83351.52089045719, Loss: 2.296365261077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4809 , TFLOPS: 103.34802885053921, Tokens per sec: 84448.33478871535, Loss: 2.3141751289367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4810 , TFLOPS: 103.27733381700861, Tokens per sec: 84390.56805696544, Loss: 2.3072283267974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4811 , TFLOPS: 102.57336733651907, Tokens per sec: 83815.33892404808, Loss: 2.3010990619659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4812 , TFLOPS: 102.58898790464248, Tokens per sec: 83828.10289237091, Loss: 2.3178045749664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4813 , TFLOPS: 103.36292001545016, Tokens per sec: 84460.50274289677, Loss: 2.286257266998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4814 , TFLOPS: 102.34497603948962, Tokens per sec: 83628.71451593029, Loss: 2.300685167312622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4815 , TFLOPS: 102.73643984384779, Tokens per sec: 83948.58966764616, Loss: 2.301145553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4816 , TFLOPS: 102.52898294306786, Tokens per sec: 83779.07129360316, Loss: 2.315748691558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4817 , TFLOPS: 101.87123033000792, Tokens per sec: 83241.6047014133, Loss: 2.261476516723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4818 , TFLOPS: 103.33050492146404, Tokens per sec: 84434.01553516196, Loss: 2.309382915496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4819 , TFLOPS: 102.66346795782187, Tokens per sec: 83888.9624611109, Loss: 2.2921199798583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4820 , TFLOPS: 102.66153361979644, Tokens per sec: 83887.38186371606, Loss: 2.301332473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4821 , TFLOPS: 102.59552392426787, Tokens per sec: 83833.44364225892, Loss: 2.2984418869018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4822 , TFLOPS: 101.89462833774768, Tokens per sec: 83260.72381585576, Loss: 2.3000540733337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4823 , TFLOPS: 102.62096460226171, Tokens per sec: 83854.231875149, Loss: 2.2929866313934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4824 , TFLOPS: 101.35309973594563, Tokens per sec: 82818.22685513695, Loss: 2.3068225383758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4825 , TFLOPS: 103.43296865040325, Tokens per sec: 84517.74128572922, Loss: 2.2906267642974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4826 , TFLOPS: 102.00619604739533, Tokens per sec: 83351.88866341721, Loss: 2.2992000579833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4827 , TFLOPS: 101.96811208438784, Tokens per sec: 83320.76927687536, Loss: 2.3031368255615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4828 , TFLOPS: 102.14563782877191, Tokens per sec: 83465.8301325308, Loss: 2.315033197402954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4829 , TFLOPS: 102.61364093679263, Tokens per sec: 83848.2475195663, Loss: 2.3156204223632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4830 , TFLOPS: 100.91819114496934, Tokens per sec: 82462.85184990709, Loss: 2.2926218509674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4831 , TFLOPS: 102.81984634488005, Tokens per sec: 84016.74326671397, Loss: 2.292768716812134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4832 , TFLOPS: 102.92292443221828, Tokens per sec: 84100.97102534406, Loss: 2.2746245861053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4833 , TFLOPS: 99.83457981690637, Tokens per sec: 81577.40513910969, Loss: 2.3023335933685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4834 , TFLOPS: 103.3979044098542, Tokens per sec: 84489.08939214275, Loss: 2.2847113609313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4835 , TFLOPS: 100.83748089229206, Tokens per sec: 82396.90142477775, Loss: 2.3020548820495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4836 , TFLOPS: 102.7318750740491, Tokens per sec: 83944.85967673601, Loss: 2.2762413024902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4837 , TFLOPS: 101.51418206239298, Tokens per sec: 82949.85137070503, Loss: 2.307063579559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4838 , TFLOPS: 102.12808478129863, Tokens per sec: 83451.48708558473, Loss: 2.2908926010131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4839 , TFLOPS: 101.80646411502855, Tokens per sec: 83188.68255992296, Loss: 2.2790989875793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4840 , TFLOPS: 102.55647532039038, Tokens per sec: 83801.53602283013, Loss: 2.311286449432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4841 , TFLOPS: 102.03725359616908, Tokens per sec: 83377.26658601267, Loss: 2.272726535797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4842 , TFLOPS: 102.1328943799072, Tokens per sec: 83455.41713241788, Loss: 2.312859058380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4843 , TFLOPS: 101.38644533089264, Tokens per sec: 82845.47439916011, Loss: 2.310434103012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4844 , TFLOPS: 102.26335775753385, Tokens per sec: 83562.02211670273, Loss: 2.2889821529388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4845 , TFLOPS: 102.59038463095568, Tokens per sec: 83829.24419339634, Loss: 2.2921626567840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4846 , TFLOPS: 102.07468793439652, Tokens per sec: 83407.85514742389, Loss: 2.315798759460449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4847 , TFLOPS: 103.23504226507484, Tokens per sec: 84356.01054119895, Loss: 2.312681198120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4848 , TFLOPS: 102.58595614418383, Tokens per sec: 83825.62556285568, Loss: 2.30434513092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4849 , TFLOPS: 102.63858207487546, Tokens per sec: 83868.62756553579, Loss: 2.296560287475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4850 , TFLOPS: 103.36891585525214, Tokens per sec: 84465.40209794563, Loss: 2.307424545288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4851 , TFLOPS: 103.28065347043696, Tokens per sec: 84393.28063123721, Loss: 2.310243606567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4852 , TFLOPS: 102.69767380684438, Tokens per sec: 83916.9129408841, Loss: 2.298856258392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4853 , TFLOPS: 102.63143999288674, Tokens per sec: 83862.79158648926, Loss: 2.318906307220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4854 , TFLOPS: 101.72621486225188, Tokens per sec: 83123.1088296796, Loss: 2.30568790435791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4855 , TFLOPS: 102.50642472103969, Tokens per sec: 83760.63838968365, Loss: 2.2996315956115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4856 , TFLOPS: 103.360606794733, Tokens per sec: 84458.61255070117, Loss: 2.2942543029785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4857 , TFLOPS: 102.72270739231513, Tokens per sec: 83937.36852799407, Loss: 2.292522430419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4858 , TFLOPS: 101.83921861317135, Tokens per sec: 83215.44710353123, Loss: 2.2970073223114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4859 , TFLOPS: 103.15761944789584, Tokens per sec: 84292.74636424136, Loss: 2.313288450241089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4860 , TFLOPS: 101.86905406083237, Tokens per sec: 83239.8264158478, Loss: 2.3038785457611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4861 , TFLOPS: 102.55641960079473, Tokens per sec: 83801.49049291418, Loss: 2.3153553009033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4862 , TFLOPS: 101.49882063595265, Tokens per sec: 82937.29915372236, Loss: 2.306676149368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4863 , TFLOPS: 103.25352279822265, Tokens per sec: 84371.11146056527, Loss: 2.277784824371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4864 , TFLOPS: 101.85247852874562, Tokens per sec: 83226.28212188758, Loss: 2.288369655609131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4865 , TFLOPS: 101.85301936058487, Tokens per sec: 83226.72404950568, Loss: 2.325911045074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4866 , TFLOPS: 102.19682081914414, Tokens per sec: 83507.65307153133, Loss: 2.3047091960906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4867 , TFLOPS: 103.38477074468703, Tokens per sec: 84478.35753624402, Loss: 2.2807297706604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4868 , TFLOPS: 100.2465941538184, Tokens per sec: 81914.07265999308, Loss: 2.2949531078338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4869 , TFLOPS: 103.3251560419363, Tokens per sec: 84429.64482800743, Loss: 2.303072929382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4870 , TFLOPS: 102.83327677178453, Tokens per sec: 84027.71761426756, Loss: 2.3126392364501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4871 , TFLOPS: 99.71701929847039, Tokens per sec: 81481.34341321868, Loss: 2.2984795570373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4872 , TFLOPS: 103.32604947839427, Tokens per sec: 84430.37487793635, Loss: 2.2876710891723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4873 , TFLOPS: 100.65670785006188, Tokens per sec: 82249.18711845938, Loss: 2.2948696613311768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4874 , TFLOPS: 102.54972099396103, Tokens per sec: 83796.01688883289, Loss: 2.30623197555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4875 , TFLOPS: 101.34042806544083, Tokens per sec: 82807.87250696965, Loss: 2.2885935306549072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4876 , TFLOPS: 102.6257889237119, Tokens per sec: 83858.17395239502, Loss: 2.290940046310425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4877 , TFLOPS: 101.30412139247056, Tokens per sec: 82778.20538986874, Loss: 2.3076798915863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4878 , TFLOPS: 102.10612158338668, Tokens per sec: 83433.54039118787, Loss: 2.2987160682678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4879 , TFLOPS: 102.03534722542867, Tokens per sec: 83375.70884140632, Loss: 2.306938648223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4880 , TFLOPS: 101.52949699062482, Tokens per sec: 82962.3655928046, Loss: 2.3106677532196045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4881 , TFLOPS: 100.75587092348837, Tokens per sec: 82330.21581844076, Loss: 2.2783658504486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4882 , TFLOPS: 102.11668804400773, Tokens per sec: 83442.17451816623, Loss: 2.2906579971313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4883 , TFLOPS: 102.7749642863142, Tokens per sec: 83980.06898129283, Loss: 2.2966299057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4884 , TFLOPS: 102.03715013052211, Tokens per sec: 83377.18204156907, Loss: 2.2874228954315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4885 , TFLOPS: 103.29002860724114, Tokens per sec: 84400.9412968574, Loss: 2.282590627670288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4886 , TFLOPS: 103.36625167476527, Tokens per sec: 84463.22512748769, Loss: 2.301365613937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4887 , TFLOPS: 102.67453136839465, Tokens per sec: 83898.00265868747, Loss: 2.2986416816711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4888 , TFLOPS: 103.38708134595325, Tokens per sec: 84480.24558801882, Loss: 2.3216943740844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4889 , TFLOPS: 102.56739372954948, Tokens per sec: 83810.4577360189, Loss: 2.2945852279663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4890 , TFLOPS: 102.75911770385324, Tokens per sec: 83967.12033083667, Loss: 2.2945947647094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4891 , TFLOPS: 102.8186529430367, Tokens per sec: 84015.76810734687, Loss: 2.3099477291107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4892 , TFLOPS: 102.56810984811635, Tokens per sec: 83811.04289493442, Loss: 2.314957857131958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4893 , TFLOPS: 102.45770480531405, Tokens per sec: 83720.82809238201, Loss: 2.305595874786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4894 , TFLOPS: 103.459316873795, Tokens per sec: 84539.27109732624, Loss: 2.3024308681488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4895 , TFLOPS: 102.04154079628097, Tokens per sec: 83380.76976758643, Loss: 2.3152823448181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4896 , TFLOPS: 102.62295219971902, Tokens per sec: 83855.85599219668, Loss: 2.2809648513793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4897 , TFLOPS: 103.23450516857058, Tokens per sec: 84355.57166581923, Loss: 2.3070626258850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4898 , TFLOPS: 101.82061251067277, Tokens per sec: 83200.24357821596, Loss: 2.2792465686798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4899 , TFLOPS: 102.59765795147814, Tokens per sec: 83835.18741082717, Loss: 2.3013882637023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4900 , TFLOPS: 101.4919629005786, Tokens per sec: 82931.69552161441, Loss: 2.30971360206604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4901 , TFLOPS: 103.36743936921098, Tokens per sec: 84464.19562319326, Loss: 2.300684690475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4902 , TFLOPS: 101.99264746970252, Tokens per sec: 83340.81777181299, Loss: 2.3192930221557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4903 , TFLOPS: 102.53437929322351, Tokens per sec: 83783.48079022975, Loss: 2.2803955078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4904 , TFLOPS: 101.47015216933394, Tokens per sec: 82913.87341165624, Loss: 2.3101582527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4905 , TFLOPS: 103.34317460105201, Tokens per sec: 84444.36825649986, Loss: 2.3051466941833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4906 , TFLOPS: 100.78190465488262, Tokens per sec: 82351.48865053088, Loss: 2.3004140853881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4907 , TFLOPS: 102.57955067111111, Tokens per sec: 83820.3914859164, Loss: 2.2733426094055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4908 , TFLOPS: 102.75149935676504, Tokens per sec: 83960.89518331719, Loss: 2.3006930351257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4909 , TFLOPS: 100.44596457427696, Tokens per sec: 82076.98336280098, Loss: 2.2915470600128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4910 , TFLOPS: 102.56102928507065, Tokens per sec: 83805.25718459986, Loss: 2.308272123336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4911 , TFLOPS: 101.30955162526656, Tokens per sec: 82782.64257287298, Loss: 2.29854416847229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4912 , TFLOPS: 101.98771824912082, Tokens per sec: 83336.78997878624, Loss: 2.3107948303222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4913 , TFLOPS: 101.96238134676416, Tokens per sec: 83316.08654364073, Loss: 2.3026225566864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4914 , TFLOPS: 102.01664321471041, Tokens per sec: 83360.42531276436, Loss: 2.3029890060424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4915 , TFLOPS: 101.91357007360348, Tokens per sec: 83276.2015958272, Loss: 2.2943336963653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4916 , TFLOPS: 101.87444293957728, Tokens per sec: 83244.22980739236, Loss: 2.308274030685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4917 , TFLOPS: 101.44401690107179, Tokens per sec: 82892.51761117757, Loss: 2.295133113861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4918 , TFLOPS: 101.48875659277904, Tokens per sec: 82929.07556497368, Loss: 2.292430877685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4919 , TFLOPS: 101.39868573218827, Tokens per sec: 82855.47634615474, Loss: 2.296480178833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4920 , TFLOPS: 101.55858368331269, Tokens per sec: 82986.13307815795, Loss: 2.2841427326202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4921 , TFLOPS: 102.22380394349493, Tokens per sec: 83529.70167704577, Loss: 2.3025946617126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4922 , TFLOPS: 102.04949941618499, Tokens per sec: 83387.27296078311, Loss: 2.3068323135375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4923 , TFLOPS: 103.33713480667187, Tokens per sec: 84439.43298503371, Loss: 2.2896475791931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4924 , TFLOPS: 103.35487389308858, Tokens per sec: 84453.92804918952, Loss: 2.3058204650878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4925 , TFLOPS: 102.6866229301107, Tokens per sec: 83907.88298503014, Loss: 2.305436849594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4926 , TFLOPS: 103.39394909330878, Tokens per sec: 84485.85740117457, Loss: 2.304326057434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4927 , TFLOPS: 103.304924901994, Tokens per sec: 84413.11344276472, Loss: 2.304417133331299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4928 , TFLOPS: 102.7570980893287, Tokens per sec: 83965.47005181915, Loss: 2.317186117172241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4929 , TFLOPS: 102.76792272784522, Tokens per sec: 83974.31514260199, Loss: 2.3152740001678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4930 , TFLOPS: 101.21225353384516, Tokens per sec: 82703.13779769687, Loss: 2.2930257320404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4931 , TFLOPS: 102.53500574293813, Tokens per sec: 83783.99267841782, Loss: 2.319322109222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4932 , TFLOPS: 103.4439869297343, Tokens per sec: 84526.74460540636, Loss: 2.313643455505371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4933 , TFLOPS: 102.87873702486104, Tokens per sec: 84064.86435730741, Loss: 2.272848606109619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4934 , TFLOPS: 102.09308003198869, Tokens per sec: 83422.88380382117, Loss: 2.284665822982788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4935 , TFLOPS: 102.51738312518361, Tokens per sec: 83769.59278380407, Loss: 2.2994437217712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4936 , TFLOPS: 101.59343142228809, Tokens per sec: 83014.60806273535, Loss: 2.3073761463165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4937 , TFLOPS: 102.08131881568451, Tokens per sec: 83413.27341121838, Loss: 2.296696901321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4938 , TFLOPS: 101.39191755896084, Tokens per sec: 82849.94589758234, Loss: 2.295290231704712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4939 , TFLOPS: 103.29190223857142, Tokens per sec: 84402.47228924895, Loss: 2.284597635269165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4940 , TFLOPS: 101.44359969852658, Tokens per sec: 82892.17670423813, Loss: 2.3019256591796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4941 , TFLOPS: 102.66686310119225, Tokens per sec: 83891.73672015763, Loss: 2.3029112815856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4942 , TFLOPS: 101.47094601253652, Tokens per sec: 82914.5220813724, Loss: 2.3080642223358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4943 , TFLOPS: 102.7732617941192, Tokens per sec: 83978.67783109425, Loss: 2.2942142486572266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4944 , TFLOPS: 99.97116139945666, Tokens per sec: 81689.00946613426, Loss: 2.3372397422790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4945 , TFLOPS: 102.66235999864263, Tokens per sec: 83888.05711914416, Loss: 2.29404878616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4946 , TFLOPS: 102.71650501332316, Tokens per sec: 83932.3004044563, Loss: 2.3004894256591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4947 , TFLOPS: 99.38155937131727, Tokens per sec: 81207.2304712351, Loss: 2.321911096572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4948 , TFLOPS: 103.2362212640851, Tokens per sec: 84356.97393164037, Loss: 2.302687644958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4949 , TFLOPS: 100.03584435369413, Tokens per sec: 81741.86357312906, Loss: 2.304762363433838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4950 , TFLOPS: 102.68095247204953, Tokens per sec: 83903.24950777777, Loss: 2.2973763942718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4951 , TFLOPS: 100.75310218083762, Tokens per sec: 82327.95340754707, Loss: 2.2913622856140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4952 , TFLOPS: 102.08002116116302, Tokens per sec: 83412.21306430426, Loss: 2.2795445919036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4953 , TFLOPS: 101.27330398989493, Tokens per sec: 82753.02369691366, Loss: 2.2991292476654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4954 , TFLOPS: 102.13557928217237, Tokens per sec: 83457.6110351743, Loss: 2.2969088554382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4955 , TFLOPS: 101.56625875706207, Tokens per sec: 82992.40457849254, Loss: 2.29699969291687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4956 , TFLOPS: 101.53499152946972, Tokens per sec: 82966.85532193679, Loss: 2.2837958335876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4957 , TFLOPS: 101.36788696469232, Tokens per sec: 82830.30988040299, Loss: 2.302192211151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4958 , TFLOPS: 102.20317921906721, Tokens per sec: 83512.84868378825, Loss: 2.285405397415161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4959 , TFLOPS: 102.85321051487585, Tokens per sec: 84044.00598888767, Loss: 2.2784297466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4960 , TFLOPS: 102.10486774919384, Tokens per sec: 83432.51585099233, Loss: 2.2830557823181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4961 , TFLOPS: 102.84437990266659, Tokens per sec: 84036.7902683313, Loss: 2.2771286964416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4962 , TFLOPS: 103.45223033515423, Tokens per sec: 84533.48050417913, Loss: 2.3120148181915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4963 , TFLOPS: 102.74601599137729, Tokens per sec: 83956.41458430448, Loss: 2.294736385345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4964 , TFLOPS: 103.17985765651929, Tokens per sec: 84310.91777697002, Loss: 2.314006805419922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4965 , TFLOPS: 101.72318934229051, Tokens per sec: 83120.63659943534, Loss: 2.3020191192626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4966 , TFLOPS: 102.05320714852769, Tokens per sec: 83390.30264432586, Loss: 2.3166823387145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4967 , TFLOPS: 102.8250933173257, Tokens per sec: 84021.03070296839, Loss: 2.306040048599243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4968 , TFLOPS: 101.32697442357082, Tokens per sec: 82796.87918987012, Loss: 2.2692322731018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4969 , TFLOPS: 102.02385674842255, Tokens per sec: 83366.31967685343, Loss: 2.300314426422119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4970 , TFLOPS: 103.34085864572482, Tokens per sec: 84442.47582978402, Loss: 2.296450614929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4971 , TFLOPS: 101.98445146941724, Tokens per sec: 83334.12060899606, Loss: 2.30889892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4972 , TFLOPS: 101.4023690889143, Tokens per sec: 82858.48611175362, Loss: 2.2910263538360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4973 , TFLOPS: 102.82425220732584, Tokens per sec: 84020.34341033541, Loss: 2.292752981185913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4974 , TFLOPS: 102.12371571911785, Tokens per sec: 83447.91701240715, Loss: 2.2937822341918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4975 , TFLOPS: 102.14839281906715, Tokens per sec: 83468.0813060208, Loss: 2.3075826168060303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4976 , TFLOPS: 100.34914986259898, Tokens per sec: 81997.87356965651, Loss: 2.3025972843170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4977 , TFLOPS: 102.76316826655274, Tokens per sec: 83970.43014988919, Loss: 2.2946016788482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4978 , TFLOPS: 101.57169606083458, Tokens per sec: 82996.8475393737, Loss: 2.2980895042419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4979 , TFLOPS: 101.5962948087721, Tokens per sec: 83016.94780954157, Loss: 2.288133144378662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4980 , TFLOPS: 101.41729405688092, Tokens per sec: 82870.681687282, Loss: 2.300416946411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4981 , TFLOPS: 101.62070592674081, Tokens per sec: 83036.89476243188, Loss: 2.2999072074890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4982 , TFLOPS: 100.61999221186296, Tokens per sec: 82219.18582534243, Loss: 2.301070213317871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4983 , TFLOPS: 102.02468729752849, Tokens per sec: 83366.99833990814, Loss: 2.313901901245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4984 , TFLOPS: 101.57648855909218, Tokens per sec: 83000.76361306987, Loss: 2.295651912689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4985 , TFLOPS: 99.61756116558011, Tokens per sec: 81400.07361255391, Loss: 2.2994518280029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4986 , TFLOPS: 101.9706585380473, Tokens per sec: 83322.85004971205, Loss: 2.275394916534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4987 , TFLOPS: 99.88410493845255, Tokens per sec: 81617.87339081512, Loss: 2.2913403511047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4988 , TFLOPS: 100.54984078294493, Tokens per sec: 82161.86328691508, Loss: 2.2879364490509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4989 , TFLOPS: 100.98973001837837, Tokens per sec: 82521.30810494405, Loss: 2.286311149597168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4990 , TFLOPS: 102.32218764082778, Tokens per sec: 83610.09352875812, Loss: 2.2922866344451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4991 , TFLOPS: 100.38176457039505, Tokens per sec: 82024.52388697396, Loss: 2.290414333343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4992 , TFLOPS: 101.59592614102033, Tokens per sec: 83016.64656163123, Loss: 2.3155784606933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4993 , TFLOPS: 101.4855767291969, Tokens per sec: 82926.47721658373, Loss: 2.315394163131714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4994 , TFLOPS: 100.93972220653053, Tokens per sec: 82480.44543456758, Loss: 2.3389670848846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4995 , TFLOPS: 101.68606294426958, Tokens per sec: 83090.29966389395, Loss: 2.298532247543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4996 , TFLOPS: 101.77416363988114, Tokens per sec: 83162.2890101918, Loss: 2.2947306632995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4997 , TFLOPS: 102.77287773982046, Tokens per sec: 83978.3640104403, Loss: 2.2922372817993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4998 , TFLOPS: 101.52173524207653, Tokens per sec: 82956.02326826061, Loss: 2.2969977855682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4999 , TFLOPS: 102.72249498384907, Tokens per sec: 83937.19496356857, Loss: 2.3159124851226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5000 , TFLOPS: 102.02784081927079, Tokens per sec: 83369.57516370264, Loss: 2.3056564331054688 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/optimizer.pt +[rank0]:[2024-08-27 20:19:30,590] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007754199992632493, 'preprocessing_with_comm': 0.001529252011096105, 'state_converting': 3.3854637650074437, : 3.3964244870003313}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000, took 16.06s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003000 + eval ppl=7.56894588470459, eval loss=2.0240538120269775 +------------------------------------------------------------------ +iteration: 5001 , TFLOPS: 101.85796533170804, Tokens per sec: 83230.76552983103, Loss: 2.314014434814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5002 , TFLOPS: 99.2917577941691, Tokens per sec: 81133.85129084908, Loss: 2.2884106636047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5003 , TFLOPS: 101.42495020480867, Tokens per sec: 82876.93772284054, Loss: 2.2971432209014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5004 , TFLOPS: 101.43844145208804, Tokens per sec: 82887.96176828812, Loss: 2.278373956680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5005 , TFLOPS: 99.20184872942933, Tokens per sec: 81060.38427958499, Loss: 2.2854714393615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5006 , TFLOPS: 102.78328595535477, Tokens per sec: 83986.86883128471, Loss: 2.3074123859405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5007 , TFLOPS: 101.57358229254798, Tokens per sec: 82998.38882786265, Loss: 2.318939447402954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5008 , TFLOPS: 101.77801385376323, Tokens per sec: 83165.43511906819, Loss: 2.29595685005188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5009 , TFLOPS: 102.55954181447561, Tokens per sec: 83804.04173408575, Loss: 2.2965943813323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5010 , TFLOPS: 103.35486087318705, Tokens per sec: 84453.91741029281, Loss: 2.2938477993011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5011 , TFLOPS: 102.68516681657375, Tokens per sec: 83906.6931571754, Loss: 2.315347909927368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5012 , TFLOPS: 102.70953427385327, Tokens per sec: 83926.60443379263, Loss: 2.3032853603363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5013 , TFLOPS: 102.01484745756892, Tokens per sec: 83358.95795337715, Loss: 2.2943177223205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5014 , TFLOPS: 101.8617512381369, Tokens per sec: 83233.85909143195, Loss: 2.2958197593688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5015 , TFLOPS: 103.18342631703918, Tokens per sec: 84313.8338213463, Loss: 2.313528060913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5016 , TFLOPS: 102.67377426441567, Tokens per sec: 83897.38400953665, Loss: 2.296734571456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5017 , TFLOPS: 101.96041705301118, Tokens per sec: 83314.48146864977, Loss: 2.301731586456299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5018 , TFLOPS: 103.3595076501143, Tokens per sec: 84457.71441133875, Loss: 2.3163907527923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5019 , TFLOPS: 101.31648073073798, Tokens per sec: 82788.30452332471, Loss: 2.27447772026062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5020 , TFLOPS: 103.27261520827915, Tokens per sec: 84386.7123603048, Loss: 2.30320405960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5021 , TFLOPS: 103.30962536561336, Tokens per sec: 84416.95431258877, Loss: 2.2798187732696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5022 , TFLOPS: 101.96029156742243, Tokens per sec: 83314.3789311449, Loss: 2.3194687366485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5023 , TFLOPS: 100.64762543135072, Tokens per sec: 82241.76564032823, Loss: 2.2733306884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5024 , TFLOPS: 102.69165628875803, Tokens per sec: 83911.99587195105, Loss: 2.323212146759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5025 , TFLOPS: 103.34999787276308, Tokens per sec: 84449.9437274616, Loss: 2.312330484390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5026 , TFLOPS: 102.70621000694716, Tokens per sec: 83923.88808972941, Loss: 2.3000476360321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5027 , TFLOPS: 99.8882334368522, Tokens per sec: 81621.24689313446, Loss: 2.3036861419677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5028 , TFLOPS: 103.41427355449201, Tokens per sec: 84502.46504160558, Loss: 2.3004534244537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5029 , TFLOPS: 102.64597389101526, Tokens per sec: 83874.66760878588, Loss: 2.297471523284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5030 , TFLOPS: 99.73662736549042, Tokens per sec: 81497.36566953729, Loss: 2.2945172786712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5031 , TFLOPS: 103.29491515146829, Tokens per sec: 84404.93421793624, Loss: 2.279552936553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5032 , TFLOPS: 101.97437272557613, Tokens per sec: 83325.88500795355, Loss: 2.2994539737701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5033 , TFLOPS: 100.92899476104152, Tokens per sec: 82471.67976270951, Loss: 2.2873523235321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5034 , TFLOPS: 101.49449459041733, Tokens per sec: 82933.76423055312, Loss: 2.314331293106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5035 , TFLOPS: 102.21049575820834, Tokens per sec: 83518.82721626492, Loss: 2.2968854904174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5036 , TFLOPS: 100.87034033645993, Tokens per sec: 82423.75172248467, Loss: 2.2908005714416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5037 , TFLOPS: 102.26012128253862, Tokens per sec: 83559.37750966985, Loss: 2.2944231033325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5038 , TFLOPS: 100.30102674680478, Tokens per sec: 81958.55093294186, Loss: 2.2956693172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5039 , TFLOPS: 102.0939116310763, Tokens per sec: 83423.56332484292, Loss: 2.315937042236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5040 , TFLOPS: 99.99907221423305, Tokens per sec: 81711.81611137633, Loss: 2.320094585418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5041 , TFLOPS: 100.59274723469309, Tokens per sec: 82196.92325314882, Loss: 2.308867931365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5042 , TFLOPS: 101.56671905151862, Tokens per sec: 82992.78069694202, Loss: 2.311981201171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5043 , TFLOPS: 99.92789363542909, Tokens per sec: 81653.65426233599, Loss: 2.302764654159546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5044 , TFLOPS: 102.139295215103, Tokens per sec: 83460.64741962853, Loss: 2.2739741802215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5045 , TFLOPS: 102.43997610377679, Tokens per sec: 83706.34151397854, Loss: 2.2870047092437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5046 , TFLOPS: 100.95190859444803, Tokens per sec: 82490.40324584051, Loss: 2.2772867679595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5047 , TFLOPS: 103.27123385873347, Tokens per sec: 84385.5836240319, Loss: 2.3082737922668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5048 , TFLOPS: 102.64888666001244, Tokens per sec: 83877.04770731484, Loss: 2.3016207218170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5049 , TFLOPS: 103.449631836443, Tokens per sec: 84531.35720399088, Loss: 2.3131752014160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5050 , TFLOPS: 102.75227490155005, Tokens per sec: 83961.52890092532, Loss: 2.3090174198150635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5051 , TFLOPS: 101.38370504688471, Tokens per sec: 82843.23524255605, Loss: 2.2839529514312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5052 , TFLOPS: 101.84232902058166, Tokens per sec: 83217.98869749521, Loss: 2.3095901012420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5053 , TFLOPS: 103.36270768610848, Tokens per sec: 84460.32924312583, Loss: 2.26470947265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5054 , TFLOPS: 102.15045066016829, Tokens per sec: 83469.76282095807, Loss: 2.3044168949127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5055 , TFLOPS: 101.43406036777225, Tokens per sec: 82884.38187151455, Loss: 2.304642915725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5056 , TFLOPS: 103.32412839032524, Tokens per sec: 84428.80510742194, Loss: 2.2883944511413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5057 , TFLOPS: 101.2864272869036, Tokens per sec: 82763.74708072317, Loss: 2.3153076171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5058 , TFLOPS: 103.32536837029294, Tokens per sec: 84429.81832697346, Loss: 2.2768335342407227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5059 , TFLOPS: 103.33589276839785, Tokens per sec: 84438.41808358711, Loss: 2.3123533725738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5060 , TFLOPS: 101.89842080489045, Tokens per sec: 83263.82273838492, Loss: 2.2901065349578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5061 , TFLOPS: 101.48199626115748, Tokens per sec: 82923.55152397929, Loss: 2.3011882305145264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5062 , TFLOPS: 102.7530897874329, Tokens per sec: 83962.19476515723, Loss: 2.306736946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5063 , TFLOPS: 103.35336808350944, Tokens per sec: 84452.6976134194, Loss: 2.2831528186798096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5064 , TFLOPS: 102.68181125443115, Tokens per sec: 83903.95124096883, Loss: 2.309197425842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5065 , TFLOPS: 99.6737376514784, Tokens per sec: 81445.9768652928, Loss: 2.306593894958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5066 , TFLOPS: 103.46703177209034, Tokens per sec: 84545.57513932251, Loss: 2.305476427078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5067 , TFLOPS: 103.38829702019277, Tokens per sec: 84481.23894673417, Loss: 2.2766432762145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5068 , TFLOPS: 99.70481385564852, Tokens per sec: 81471.37003169289, Loss: 2.2935845851898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5069 , TFLOPS: 103.36483443483584, Tokens per sec: 84462.06706425836, Loss: 2.309061050415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5070 , TFLOPS: 102.0047703002846, Tokens per sec: 83350.72364875104, Loss: 2.275451898574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5071 , TFLOPS: 100.02649036434504, Tokens per sec: 81734.2201876388, Loss: 2.306882619857788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5072 , TFLOPS: 102.00694123185568, Tokens per sec: 83352.49757282251, Loss: 2.2891526222229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5073 , TFLOPS: 102.32438234915867, Tokens per sec: 83611.8868814321, Loss: 2.2925360202789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5074 , TFLOPS: 100.61152051095877, Tokens per sec: 82212.26338045264, Loss: 2.3062121868133545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5075 , TFLOPS: 101.39639022614867, Tokens per sec: 82853.6006290782, Loss: 2.3144545555114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5076 , TFLOPS: 100.74522096395651, Tokens per sec: 82321.51346235294, Loss: 2.3107500076293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5077 , TFLOPS: 102.0208213740613, Tokens per sec: 83363.83939432536, Loss: 2.2930445671081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5078 , TFLOPS: 100.3032471590243, Tokens per sec: 81960.36528892492, Loss: 2.292168140411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5079 , TFLOPS: 100.93429312748114, Tokens per sec: 82476.0091943194, Loss: 2.32315731048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5080 , TFLOPS: 101.70984043286558, Tokens per sec: 83109.72886191274, Loss: 2.2951853275299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5081 , TFLOPS: 100.0508110395464, Tokens per sec: 81754.09323741571, Loss: 2.2930808067321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5082 , TFLOPS: 102.15937796304674, Tokens per sec: 83477.05754994927, Loss: 2.307018756866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5083 , TFLOPS: 102.24980717635516, Tokens per sec: 83550.94958799864, Loss: 2.3134689331054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5084 , TFLOPS: 101.77110020125077, Tokens per sec: 83159.785795627, Loss: 2.273334503173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5085 , TFLOPS: 102.52832145539136, Tokens per sec: 83778.53077499448, Loss: 2.29549241065979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5086 , TFLOPS: 103.38217745748241, Tokens per sec: 84476.23849451198, Loss: 2.3013508319854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5087 , TFLOPS: 103.46922529245532, Tokens per sec: 84547.36752127897, Loss: 2.2895641326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5088 , TFLOPS: 102.88437674080461, Tokens per sec: 84069.47271438409, Loss: 2.305398941040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5089 , TFLOPS: 102.00363249949152, Tokens per sec: 83349.79392243341, Loss: 2.290809392929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5090 , TFLOPS: 101.93146846260095, Tokens per sec: 83290.82683022255, Loss: 2.3102896213531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5091 , TFLOPS: 103.37144383573342, Tokens per sec: 84467.46777587288, Loss: 2.3066341876983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5092 , TFLOPS: 102.01658074938983, Tokens per sec: 83360.37427074292, Loss: 2.2817864418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5093 , TFLOPS: 101.33312246222792, Tokens per sec: 82801.90291052141, Loss: 2.292862892150879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5094 , TFLOPS: 102.54917447986038, Tokens per sec: 83795.57031809271, Loss: 2.2687013149261475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5095 , TFLOPS: 101.50551469465132, Tokens per sec: 82942.76904140545, Loss: 2.2937679290771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5096 , TFLOPS: 102.78242988302753, Tokens per sec: 83986.16931254885, Loss: 2.302860736846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5097 , TFLOPS: 102.74246697058989, Tokens per sec: 83953.51458805916, Loss: 2.2975568771362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5098 , TFLOPS: 102.02535749202333, Tokens per sec: 83367.5459730822, Loss: 2.277897357940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5099 , TFLOPS: 101.2343207127643, Tokens per sec: 82721.16945765141, Loss: 2.294865369796753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5100 , TFLOPS: 102.71984071094509, Tokens per sec: 83935.02608885166, Loss: 2.291473865509033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5101 , TFLOPS: 103.22460503914523, Tokens per sec: 84347.48201521377, Loss: 2.2840752601623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5102 , TFLOPS: 101.90311342094151, Tokens per sec: 83267.6571957589, Loss: 2.2927749156951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5103 , TFLOPS: 101.08365143432141, Tokens per sec: 82598.05371166344, Loss: 2.294727087020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5104 , TFLOPS: 102.79643636669337, Tokens per sec: 83997.61437091118, Loss: 2.3040709495544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5105 , TFLOPS: 103.37077728273029, Tokens per sec: 84466.92311825542, Loss: 2.2823755741119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5106 , TFLOPS: 99.08605402554745, Tokens per sec: 80965.76544622224, Loss: 2.292381763458252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5107 , TFLOPS: 103.42217831472419, Tokens per sec: 84508.92422467792, Loss: 2.2890634536743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5108 , TFLOPS: 102.03626096086285, Tokens per sec: 83376.45547815123, Loss: 2.288376569747925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5109 , TFLOPS: 100.80188649371246, Tokens per sec: 82367.81632541704, Loss: 2.276406764984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5110 , TFLOPS: 102.00033748524513, Tokens per sec: 83347.10148147147, Loss: 2.294468879699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5111 , TFLOPS: 101.52971858598747, Tokens per sec: 82962.5466640798, Loss: 2.2972617149353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5112 , TFLOPS: 100.77807888921171, Tokens per sec: 82348.3625189172, Loss: 2.295583486557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5113 , TFLOPS: 101.55827111750547, Tokens per sec: 82985.87767259071, Loss: 2.2899279594421387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5114 , TFLOPS: 101.52584354560445, Tokens per sec: 82959.38026883041, Loss: 2.2841968536376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5115 , TFLOPS: 102.1884429672635, Tokens per sec: 83500.80731309456, Loss: 2.310732364654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5116 , TFLOPS: 101.16411771667654, Tokens per sec: 82663.8047823622, Loss: 2.3102824687957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5117 , TFLOPS: 101.48919609962769, Tokens per sec: 82929.43469733362, Loss: 2.2906033992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5118 , TFLOPS: 101.62449003262296, Tokens per sec: 83039.98685275973, Loss: 2.2987782955169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5119 , TFLOPS: 99.91761712735348, Tokens per sec: 81645.25706304655, Loss: 2.2915499210357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5120 , TFLOPS: 101.77937057784943, Tokens per sec: 83166.54373324421, Loss: 2.300701856613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5121 , TFLOPS: 100.91109162577939, Tokens per sec: 82457.05065001911, Loss: 2.2965967655181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5122 , TFLOPS: 101.71378033805726, Tokens per sec: 83112.94825986706, Loss: 2.3004519939422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5123 , TFLOPS: 103.23084653986282, Tokens per sec: 84352.58210612064, Loss: 2.309767007827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5124 , TFLOPS: 102.55050089032119, Tokens per sec: 83796.65416222508, Loss: 2.3074488639831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5125 , TFLOPS: 102.70493181335263, Tokens per sec: 83922.8436448397, Loss: 2.3116326332092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5126 , TFLOPS: 103.37969491934084, Tokens per sec: 84474.20994869027, Loss: 2.282684326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5127 , TFLOPS: 100.54144137296898, Tokens per sec: 82154.99991280367, Loss: 2.301790475845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5128 , TFLOPS: 102.70526483314592, Tokens per sec: 83923.1157638855, Loss: 2.308417797088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5129 , TFLOPS: 102.76526706049194, Tokens per sec: 83972.14512844467, Loss: 2.3115031719207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5130 , TFLOPS: 102.63225061487312, Tokens per sec: 83863.4539665816, Loss: 2.3010618686676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5131 , TFLOPS: 99.11576432592955, Tokens per sec: 80990.04249747563, Loss: 2.282212018966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5132 , TFLOPS: 103.37263316012049, Tokens per sec: 84468.43960344548, Loss: 2.3012235164642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5133 , TFLOPS: 102.23747547460442, Tokens per sec: 83540.87303705186, Loss: 2.306684732437134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5134 , TFLOPS: 102.00652702019124, Tokens per sec: 83352.15910980874, Loss: 2.292611598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5135 , TFLOPS: 102.02717930561458, Tokens per sec: 83369.03462386527, Loss: 2.2896268367767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5136 , TFLOPS: 101.52509357439457, Tokens per sec: 82958.76744804885, Loss: 2.301017999649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5137 , TFLOPS: 100.76760030274605, Tokens per sec: 82339.8001961733, Loss: 2.277919054031372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5138 , TFLOPS: 101.43185630634076, Tokens per sec: 82882.58087618128, Loss: 2.3164234161376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5139 , TFLOPS: 103.33900728740616, Tokens per sec: 84440.96303724364, Loss: 2.292949914932251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5140 , TFLOPS: 100.50971044488969, Tokens per sec: 82129.07175464334, Loss: 2.3036980628967285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5141 , TFLOPS: 100.95382397677288, Tokens per sec: 82491.96835404447, Loss: 2.297562599182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5142 , TFLOPS: 102.37547893593795, Tokens per sec: 83653.63921783262, Loss: 2.282166004180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5143 , TFLOPS: 102.39235380592125, Tokens per sec: 83667.42810849378, Loss: 2.28119158744812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5144 , TFLOPS: 99.87958326027655, Tokens per sec: 81614.17861117916, Loss: 2.291240692138672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5145 , TFLOPS: 102.54960229289185, Tokens per sec: 83795.91989513363, Loss: 2.286914587020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5146 , TFLOPS: 101.54385768006534, Tokens per sec: 82974.1000818235, Loss: 2.2864530086517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5147 , TFLOPS: 101.39536103029457, Tokens per sec: 82852.75964665196, Loss: 2.3000669479370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5148 , TFLOPS: 100.82687068314793, Tokens per sec: 82388.231549755, Loss: 2.309540271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5149 , TFLOPS: 102.22902582211994, Tokens per sec: 83533.96860849338, Loss: 2.298463821411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5150 , TFLOPS: 100.8049970886441, Tokens per sec: 82370.35807260951, Loss: 2.293705463409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5151 , TFLOPS: 100.98510018602342, Tokens per sec: 82517.52494974431, Loss: 2.2883639335632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5152 , TFLOPS: 101.66607435006094, Tokens per sec: 83073.96646901387, Loss: 2.284949779510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5153 , TFLOPS: 102.10606607938324, Tokens per sec: 83433.49503743785, Loss: 2.3092801570892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5154 , TFLOPS: 100.78350558045383, Tokens per sec: 82352.7968080266, Loss: 2.2954158782958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5155 , TFLOPS: 102.37486020442587, Tokens per sec: 83653.13363638654, Loss: 2.282818078994751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5156 , TFLOPS: 102.17776675095028, Tokens per sec: 83492.08350191478, Loss: 2.303678512573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5157 , TFLOPS: 101.07435447689322, Tokens per sec: 82590.4569284239, Loss: 2.3175745010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5158 , TFLOPS: 101.38780945296311, Tokens per sec: 82846.58905841953, Loss: 2.2798995971679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5159 , TFLOPS: 100.7626926690893, Tokens per sec: 82335.79004237865, Loss: 2.2997448444366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5160 , TFLOPS: 101.59339227480523, Tokens per sec: 83014.57607431937, Loss: 2.300091505050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5161 , TFLOPS: 103.43847602056735, Tokens per sec: 84522.24149966263, Loss: 2.287707805633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5162 , TFLOPS: 102.66510004040089, Tokens per sec: 83890.29607779978, Loss: 2.290570020675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5163 , TFLOPS: 101.64291772903383, Tokens per sec: 83055.04459786815, Loss: 2.2783546447753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5164 , TFLOPS: 103.35865032907437, Tokens per sec: 84457.0138722476, Loss: 2.2833218574523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5165 , TFLOPS: 101.69459426277861, Tokens per sec: 83097.27082386335, Loss: 2.282968521118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5166 , TFLOPS: 102.29323931605863, Tokens per sec: 83586.43910739309, Loss: 2.2841503620147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5167 , TFLOPS: 102.81218879976336, Tokens per sec: 84010.48608947637, Loss: 2.3219900131225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5168 , TFLOPS: 101.98252850667951, Tokens per sec: 83332.54930664161, Loss: 2.2971043586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5169 , TFLOPS: 100.65929168785908, Tokens per sec: 82251.29843883737, Loss: 2.288961172103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5170 , TFLOPS: 102.70900654069519, Tokens per sec: 83926.17320944412, Loss: 2.306584358215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5171 , TFLOPS: 102.18694457574071, Tokens per sec: 83499.58293880927, Loss: 2.292196750640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5172 , TFLOPS: 101.49374331799507, Tokens per sec: 82933.15034651743, Loss: 2.2875545024871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5173 , TFLOPS: 102.59852857758837, Tokens per sec: 83835.89882183376, Loss: 2.2748918533325195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5174 , TFLOPS: 100.31088408970709, Tokens per sec: 81966.60562157792, Loss: 2.300739049911499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5175 , TFLOPS: 101.33412728822245, Tokens per sec: 82802.72397970807, Loss: 2.2919411659240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5176 , TFLOPS: 101.47531793843903, Tokens per sec: 82918.09449457073, Loss: 2.2996559143066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5177 , TFLOPS: 102.60806098632557, Tokens per sec: 83843.68799839896, Loss: 2.2943930625915527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5178 , TFLOPS: 100.02984097158594, Tokens per sec: 81736.95805506765, Loss: 2.3112590312957764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5179 , TFLOPS: 101.08983391857562, Tokens per sec: 82603.10557869886, Loss: 2.320924997329712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5180 , TFLOPS: 101.63448640620743, Tokens per sec: 83048.15514694515, Loss: 2.2995622158050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5181 , TFLOPS: 101.6982979706833, Tokens per sec: 83100.2972189342, Loss: 2.299827814102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5182 , TFLOPS: 100.49880076222344, Tokens per sec: 82120.15717209651, Loss: 2.3088700771331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5183 , TFLOPS: 102.20438208957162, Tokens per sec: 83513.83158024204, Loss: 2.284825563430786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5184 , TFLOPS: 102.80479988201546, Tokens per sec: 84004.44841457688, Loss: 2.2918832302093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5185 , TFLOPS: 101.59399473734832, Tokens per sec: 83015.06836197208, Loss: 2.3128912448883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5186 , TFLOPS: 102.87533902659521, Tokens per sec: 84062.08776545213, Loss: 2.306062698364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5187 , TFLOPS: 102.6208073738484, Tokens per sec: 83854.10339976505, Loss: 2.291020631790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5188 , TFLOPS: 101.50871825594295, Tokens per sec: 82945.38675380385, Loss: 2.2987918853759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5189 , TFLOPS: 101.88827457891479, Tokens per sec: 83255.5319959531, Loss: 2.2685303688049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5190 , TFLOPS: 100.79877905254602, Tokens per sec: 82365.27715524728, Loss: 2.3054909706115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5191 , TFLOPS: 101.24961103117091, Tokens per sec: 82733.66357042926, Loss: 2.3155646324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5192 , TFLOPS: 101.64820206410592, Tokens per sec: 83059.36256408645, Loss: 2.3002848625183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5193 , TFLOPS: 102.2805113194512, Tokens per sec: 83576.03873371736, Loss: 2.271554708480835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5194 , TFLOPS: 102.14400847706976, Tokens per sec: 83464.49874731172, Loss: 2.2770562171936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5195 , TFLOPS: 98.71708460487365, Tokens per sec: 80664.27103447182, Loss: 2.2834951877593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5196 , TFLOPS: 102.81188587625657, Tokens per sec: 84010.23856288102, Loss: 2.2958710193634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5197 , TFLOPS: 102.3345919073908, Tokens per sec: 83620.229374281, Loss: 2.310952663421631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5198 , TFLOPS: 100.14254231018101, Tokens per sec: 81829.04922002424, Loss: 2.2888894081115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5199 , TFLOPS: 102.6970304638394, Tokens per sec: 83916.38724875361, Loss: 2.2806835174560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5200 , TFLOPS: 102.48058287080258, Tokens per sec: 83739.52234862649, Loss: 2.3123579025268555 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/model.pt +[rank0]:[2024-08-27 21:12:24,068] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007698611996602267, 'preprocessing_with_comm': 0.0015071489906404167, 'state_converting': 3.376210374990478, : 3.3870217599906027}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200, took 15.89s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003200 + eval ppl=8.100301742553711, eval loss=2.0919013023376465 +------------------------------------------------------------------ +iteration: 5201 , TFLOPS: 100.99703923048706, Tokens per sec: 82527.28065031391, Loss: 2.2900760173797607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5202 , TFLOPS: 101.09215678421165, Tokens per sec: 82605.00365200595, Loss: 2.284832000732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5203 , TFLOPS: 102.7381259703602, Tokens per sec: 83949.9674450242, Loss: 2.30566143989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5204 , TFLOPS: 101.11655746199673, Tokens per sec: 82624.94207395347, Loss: 2.305877208709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5205 , TFLOPS: 101.57975539741825, Tokens per sec: 83003.43303076201, Loss: 2.304819107055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5206 , TFLOPS: 102.18640310536023, Tokens per sec: 83499.14048942269, Loss: 2.324138879776001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5207 , TFLOPS: 102.40513485966287, Tokens per sec: 83677.87183651964, Loss: 2.31699275970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5208 , TFLOPS: 100.94620652202177, Tokens per sec: 82485.7439356766, Loss: 2.3085293769836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5209 , TFLOPS: 101.08215204217737, Tokens per sec: 82596.82851974475, Loss: 2.295362949371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5210 , TFLOPS: 103.27805137376758, Tokens per sec: 84391.15439106473, Loss: 2.292127847671509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5211 , TFLOPS: 103.22268229340717, Tokens per sec: 84345.91089017532, Loss: 2.286632537841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5212 , TFLOPS: 103.31144873594843, Tokens per sec: 84418.4442354273, Loss: 2.318622589111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5213 , TFLOPS: 102.65197444662125, Tokens per sec: 83879.5708172396, Loss: 2.2843854427337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5214 , TFLOPS: 103.01459768410486, Tokens per sec: 84175.87959934003, Loss: 2.2992031574249268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5215 , TFLOPS: 102.4660389566266, Tokens per sec: 83727.63814195966, Loss: 2.290142297744751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5216 , TFLOPS: 102.63020397755251, Tokens per sec: 83861.7816065418, Loss: 2.2976717948913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5217 , TFLOPS: 101.48827718905916, Tokens per sec: 82928.68383185321, Loss: 2.283198118209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5218 , TFLOPS: 101.93755317362911, Tokens per sec: 83295.79880423805, Loss: 2.2617335319519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5219 , TFLOPS: 102.73851027753271, Tokens per sec: 83950.28147230782, Loss: 2.289735794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5220 , TFLOPS: 102.64185010301277, Tokens per sec: 83871.29795545331, Loss: 2.301189422607422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5221 , TFLOPS: 103.24944097070497, Tokens per sec: 84367.77609422497, Loss: 2.296048879623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5222 , TFLOPS: 101.26533728676168, Tokens per sec: 82746.51389870231, Loss: 2.302736520767212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5223 , TFLOPS: 102.01094971819983, Tokens per sec: 83355.77301020193, Loss: 2.2751927375793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5224 , TFLOPS: 102.55663484083817, Tokens per sec: 83801.66637109446, Loss: 2.282958507537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5225 , TFLOPS: 102.24676764024045, Tokens per sec: 83548.46590479469, Loss: 2.2848238945007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5226 , TFLOPS: 101.24810718394407, Tokens per sec: 82732.43473814774, Loss: 2.289367437362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5227 , TFLOPS: 102.15334063172959, Tokens per sec: 83472.12429111531, Loss: 2.2875611782073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5228 , TFLOPS: 101.99550747476427, Tokens per sec: 83343.15475557202, Loss: 2.2972006797790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5229 , TFLOPS: 102.56056202406377, Tokens per sec: 83804.87537360277, Loss: 2.310497522354126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5230 , TFLOPS: 100.3139082812255, Tokens per sec: 81969.07676631722, Loss: 2.28182315826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5231 , TFLOPS: 102.80826469149683, Tokens per sec: 84007.27959959625, Loss: 2.2967095375061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5232 , TFLOPS: 102.12453650188402, Tokens per sec: 83448.58769513422, Loss: 2.291445255279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5233 , TFLOPS: 100.70802706125731, Tokens per sec: 82291.12136700118, Loss: 2.303140878677368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5234 , TFLOPS: 102.71954065110181, Tokens per sec: 83934.7809022295, Loss: 2.286724805831909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5235 , TFLOPS: 102.71740524359012, Tokens per sec: 83933.03600578141, Loss: 2.308574676513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5236 , TFLOPS: 98.63847905716388, Tokens per sec: 80600.04041795134, Loss: 2.2811052799224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5237 , TFLOPS: 102.22028245291371, Tokens per sec: 83526.82417644063, Loss: 2.318119525909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5238 , TFLOPS: 102.21377432218192, Tokens per sec: 83521.5062152855, Loss: 2.3118338584899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5239 , TFLOPS: 101.4492462044237, Tokens per sec: 82896.79060956066, Loss: 2.3022992610931396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5240 , TFLOPS: 102.01981494024615, Tokens per sec: 83363.0170113471, Loss: 2.2844674587249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5241 , TFLOPS: 101.89812591272654, Tokens per sec: 83263.58177440659, Loss: 2.2846016883850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5242 , TFLOPS: 101.33244682070035, Tokens per sec: 82801.35082643668, Loss: 2.303619623184204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5243 , TFLOPS: 101.53745746342695, Tokens per sec: 82968.87030005203, Loss: 2.2879040241241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5244 , TFLOPS: 102.02315080237997, Tokens per sec: 83365.74283016937, Loss: 2.3030214309692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5245 , TFLOPS: 102.43206244498052, Tokens per sec: 83699.87505966041, Loss: 2.310256004333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5246 , TFLOPS: 101.02188378767684, Tokens per sec: 82547.5817775495, Loss: 2.2822139263153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5247 , TFLOPS: 101.88486327767795, Tokens per sec: 83252.74453389777, Loss: 2.3185596466064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5248 , TFLOPS: 102.6373046652016, Tokens per sec: 83867.58376120783, Loss: 2.306124448776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5249 , TFLOPS: 102.70132816998958, Tokens per sec: 83919.89901508173, Loss: 2.287461042404175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5250 , TFLOPS: 103.2289450382254, Tokens per sec: 84351.0283401837, Loss: 2.3135476112365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5251 , TFLOPS: 102.06688651332664, Tokens per sec: 83401.48040544125, Loss: 2.2862725257873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5252 , TFLOPS: 103.44435462303653, Tokens per sec: 84527.0450570689, Loss: 2.2971839904785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5253 , TFLOPS: 102.62173504910558, Tokens per sec: 83854.86142709819, Loss: 2.28706693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5254 , TFLOPS: 101.99829315205244, Tokens per sec: 83345.4310041939, Loss: 2.2867093086242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5255 , TFLOPS: 102.10446425377413, Tokens per sec: 83432.18614449799, Loss: 2.300316095352173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5256 , TFLOPS: 103.3390017937503, Tokens per sec: 84440.95854823603, Loss: 2.2987828254699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5257 , TFLOPS: 102.7244225238672, Tokens per sec: 83938.77000613668, Loss: 2.29079270362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5258 , TFLOPS: 102.65910705372413, Tokens per sec: 83885.39905411299, Loss: 2.2707784175872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5259 , TFLOPS: 102.62973676837262, Tokens per sec: 83861.39983789388, Loss: 2.29306960105896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5260 , TFLOPS: 100.21144814766072, Tokens per sec: 81885.35395362295, Loss: 2.3019168376922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5261 , TFLOPS: 102.65705262444243, Tokens per sec: 83883.7203270612, Loss: 2.297438859939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5262 , TFLOPS: 102.12043161567016, Tokens per sec: 83445.2334869397, Loss: 2.300065279006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5263 , TFLOPS: 103.29476807182816, Tokens per sec: 84404.81403537613, Loss: 2.296149730682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5264 , TFLOPS: 101.56391557576595, Tokens per sec: 82990.48990473662, Loss: 2.3055062294006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5265 , TFLOPS: 102.01463144575322, Tokens per sec: 83358.7814445619, Loss: 2.2835144996643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5266 , TFLOPS: 101.2556183099385, Tokens per sec: 82738.57227255136, Loss: 2.2870378494262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5267 , TFLOPS: 102.50500386173533, Tokens per sec: 83759.47736896988, Loss: 2.3199386596679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5268 , TFLOPS: 100.98922647359312, Tokens per sec: 82520.89664553762, Loss: 2.314037799835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5269 , TFLOPS: 102.75349163924608, Tokens per sec: 83962.52312861837, Loss: 2.297612428665161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5270 , TFLOPS: 101.50345532406631, Tokens per sec: 82941.08627668752, Loss: 2.288442373275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5271 , TFLOPS: 101.2932540400608, Tokens per sec: 82769.32539646418, Loss: 2.2775120735168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5272 , TFLOPS: 102.02005093275449, Tokens per sec: 83363.20984690063, Loss: 2.302713394165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5273 , TFLOPS: 102.14957350662473, Tokens per sec: 83469.0460762177, Loss: 2.2929372787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5274 , TFLOPS: 102.09796931695931, Tokens per sec: 83426.87896443205, Loss: 2.30395245552063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5275 , TFLOPS: 101.06106644826603, Tokens per sec: 82579.59893816833, Loss: 2.313586950302124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5276 , TFLOPS: 102.6883417693232, Tokens per sec: 83909.28749279752, Loss: 2.2707595825195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5277 , TFLOPS: 101.84446516232958, Tokens per sec: 83219.73419390662, Loss: 2.286928415298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5278 , TFLOPS: 101.41396534199582, Tokens per sec: 82867.96170866063, Loss: 2.3121461868286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5279 , TFLOPS: 101.46257800305608, Tokens per sec: 82907.6843654142, Loss: 2.2763895988464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5280 , TFLOPS: 102.67652347098374, Tokens per sec: 83899.63045699429, Loss: 2.302683115005493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5281 , TFLOPS: 101.46381379763119, Tokens per sec: 82908.6941649737, Loss: 2.3132290840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5282 , TFLOPS: 102.91243320317601, Tokens per sec: 84092.39837202536, Loss: 2.2724833488464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5283 , TFLOPS: 101.68366431201807, Tokens per sec: 83088.33967973494, Loss: 2.310810089111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5284 , TFLOPS: 102.22910898554684, Tokens per sec: 83534.03656347033, Loss: 2.2997629642486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5285 , TFLOPS: 101.21010492668822, Tokens per sec: 82701.38211547885, Loss: 2.3070085048675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5286 , TFLOPS: 102.60240965217822, Tokens per sec: 83839.07014778888, Loss: 2.3055291175842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5287 , TFLOPS: 101.36891618530638, Tokens per sec: 82831.15088306123, Loss: 2.2766358852386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5288 , TFLOPS: 103.24560019953023, Tokens per sec: 84364.63770122785, Loss: 2.312565803527832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5289 , TFLOPS: 101.97793467834464, Tokens per sec: 83328.79557125352, Loss: 2.2907183170318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5290 , TFLOPS: 102.03610877440798, Tokens per sec: 83376.33112268134, Loss: 2.2870728969573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5291 , TFLOPS: 102.04919269426348, Tokens per sec: 83387.02233040534, Loss: 2.2886462211608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5292 , TFLOPS: 100.23376785079908, Tokens per sec: 81903.5919576176, Loss: 2.3181042671203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5293 , TFLOPS: 102.68817002391029, Tokens per sec: 83909.14715519952, Loss: 2.2801284790039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5294 , TFLOPS: 102.08639900090392, Tokens per sec: 83417.42456133677, Loss: 2.275580883026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5295 , TFLOPS: 102.72964033253302, Tokens per sec: 83943.03361191587, Loss: 2.3080828189849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5296 , TFLOPS: 102.18107060502145, Tokens per sec: 83494.7831661251, Loss: 2.2944588661193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5297 , TFLOPS: 101.2344863335971, Tokens per sec: 82721.30479069735, Loss: 2.2898142337799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5298 , TFLOPS: 101.51510666007421, Tokens per sec: 82950.60688327164, Loss: 2.300745964050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5299 , TFLOPS: 102.05486982612214, Tokens per sec: 83391.66126098942, Loss: 2.285360097885132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5300 , TFLOPS: 101.07142415484381, Tokens per sec: 82588.06248684412, Loss: 2.28930401802063 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5301 , TFLOPS: 101.18157316552633, Tokens per sec: 82678.0680789606, Loss: 2.301792860031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5302 , TFLOPS: 101.27969119933032, Tokens per sec: 82758.24285016472, Loss: 2.2778496742248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5303 , TFLOPS: 100.31067995909252, Tokens per sec: 81966.43882119797, Loss: 2.313448667526245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5304 , TFLOPS: 102.01632069960729, Tokens per sec: 83360.16177737134, Loss: 2.2975914478302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5305 , TFLOPS: 102.22855315248472, Tokens per sec: 83533.58237796684, Loss: 2.3044519424438477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5306 , TFLOPS: 100.91426585000065, Tokens per sec: 82459.64439034209, Loss: 2.2988317012786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5307 , TFLOPS: 102.42028124494432, Tokens per sec: 83690.24833783577, Loss: 2.2805562019348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5308 , TFLOPS: 99.7857044495364, Tokens per sec: 81537.46781826779, Loss: 2.330495595932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5309 , TFLOPS: 102.88629604815152, Tokens per sec: 84071.04102982461, Loss: 2.2846810817718506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5310 , TFLOPS: 100.95481866031271, Tokens per sec: 82492.78113557029, Loss: 2.290834426879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5311 , TFLOPS: 101.4342786698141, Tokens per sec: 82884.56025173253, Loss: 2.2887685298919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5312 , TFLOPS: 101.24375128998511, Tokens per sec: 82728.87542506318, Loss: 2.2842087745666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5313 , TFLOPS: 100.660379261541, Tokens per sec: 82252.18712328949, Loss: 2.303834915161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5314 , TFLOPS: 103.34127167280857, Tokens per sec: 84442.81332484641, Loss: 2.2708117961883545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5315 , TFLOPS: 101.16631164923237, Tokens per sec: 82665.59750113041, Loss: 2.2988815307617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5316 , TFLOPS: 101.50915457956478, Tokens per sec: 82945.7432850672, Loss: 2.303147792816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5317 , TFLOPS: 100.79442171274894, Tokens per sec: 82361.71666073118, Loss: 2.2788994312286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5318 , TFLOPS: 102.81171707490148, Tokens per sec: 84010.10063094845, Loss: 2.2805368900299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5319 , TFLOPS: 101.85638742814348, Tokens per sec: 83229.47618420958, Loss: 2.2789735794067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5320 , TFLOPS: 102.75231169881957, Tokens per sec: 83961.5589689215, Loss: 2.294276237487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5321 , TFLOPS: 102.89022260129767, Tokens per sec: 84074.2495174788, Loss: 2.3071582317352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5322 , TFLOPS: 101.97043101788732, Tokens per sec: 83322.66413713244, Loss: 2.293017864227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5323 , TFLOPS: 102.0446067953481, Tokens per sec: 83383.27507435004, Loss: 2.2924251556396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5324 , TFLOPS: 101.37822679211601, Tokens per sec: 82838.75881956173, Loss: 2.285536289215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5325 , TFLOPS: 101.52887929389301, Tokens per sec: 82961.86085690411, Loss: 2.2932052612304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5326 , TFLOPS: 103.23179464236576, Tokens per sec: 84353.35682508211, Loss: 2.288248300552368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5327 , TFLOPS: 101.92665719452513, Tokens per sec: 83286.89541922473, Loss: 2.296537399291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5328 , TFLOPS: 101.54883883292507, Tokens per sec: 82978.17031005146, Loss: 2.306680917739868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5329 , TFLOPS: 101.45190647488643, Tokens per sec: 82898.96438503721, Loss: 2.277803659439087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5330 , TFLOPS: 102.06219638993943, Tokens per sec: 83397.64798488705, Loss: 2.2928764820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5331 , TFLOPS: 102.08416224138615, Tokens per sec: 83415.59684755548, Loss: 2.2808361053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5332 , TFLOPS: 100.25291584989442, Tokens per sec: 81919.23828059174, Loss: 2.2988333702087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5333 , TFLOPS: 102.60286357174431, Tokens per sec: 83839.44105715126, Loss: 2.291548728942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5334 , TFLOPS: 101.41250897826325, Tokens per sec: 82866.77167636462, Loss: 2.2984628677368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5335 , TFLOPS: 101.39003865070148, Tokens per sec: 82848.4105932762, Loss: 2.30706524848938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5336 , TFLOPS: 101.2422138978621, Tokens per sec: 82727.61918238348, Loss: 2.2805495262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5337 , TFLOPS: 100.87313351807951, Tokens per sec: 82426.03410308892, Loss: 2.2934844493865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5338 , TFLOPS: 102.17540043215715, Tokens per sec: 83490.14992191445, Loss: 2.2827961444854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5339 , TFLOPS: 101.99861431534683, Tokens per sec: 83345.69343498923, Loss: 2.309434413909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5340 , TFLOPS: 100.39219361223381, Tokens per sec: 82033.04572552782, Loss: 2.2999520301818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5341 , TFLOPS: 102.22550382211942, Tokens per sec: 83531.09069162862, Loss: 2.304110527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5342 , TFLOPS: 100.8413557749848, Tokens per sec: 82400.06769117435, Loss: 2.3070757389068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5343 , TFLOPS: 100.80179818149351, Tokens per sec: 82367.74416322957, Loss: 2.295658826828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5344 , TFLOPS: 101.63266345243026, Tokens per sec: 83046.66556448682, Loss: 2.316148042678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5345 , TFLOPS: 101.10599708509255, Tokens per sec: 82616.31291813678, Loss: 2.283160448074341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5346 , TFLOPS: 101.70713910861504, Tokens per sec: 83107.52154032934, Loss: 2.3045237064361572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5347 , TFLOPS: 101.78555038435951, Tokens per sec: 83171.59341222596, Loss: 2.2845938205718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5348 , TFLOPS: 100.8105713706475, Tokens per sec: 82374.91296192928, Loss: 2.319976568222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5349 , TFLOPS: 101.34218921323969, Tokens per sec: 82809.31158617207, Loss: 2.300234079360962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5350 , TFLOPS: 102.64721812788557, Tokens per sec: 83875.68430676212, Loss: 2.2713496685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5351 , TFLOPS: 102.40002352340345, Tokens per sec: 83673.69523208457, Loss: 2.317254066467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5352 , TFLOPS: 103.2610556206652, Tokens per sec: 84377.26672369518, Loss: 2.2794554233551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5353 , TFLOPS: 102.06094050098841, Tokens per sec: 83396.62176569572, Loss: 2.3032660484313965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5354 , TFLOPS: 102.68600250703642, Tokens per sec: 83907.37602136502, Loss: 2.308178424835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5355 , TFLOPS: 101.44476262411114, Tokens per sec: 82893.12696066963, Loss: 2.28890061378479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5356 , TFLOPS: 101.3337048622636, Tokens per sec: 82802.3788045829, Loss: 2.3014612197875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5357 , TFLOPS: 102.82224092264528, Tokens per sec: 84018.69993784784, Loss: 2.261685609817505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5358 , TFLOPS: 102.18860607956911, Tokens per sec: 83500.94059635836, Loss: 2.289340019226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5359 , TFLOPS: 102.84821238317362, Tokens per sec: 84039.92188681042, Loss: 2.3057470321655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5360 , TFLOPS: 101.51296203564459, Tokens per sec: 82948.85445544265, Loss: 2.2987864017486572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5361 , TFLOPS: 101.21767284994607, Tokens per sec: 82707.56606038852, Loss: 2.2882919311523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5362 , TFLOPS: 102.7124263992071, Tokens per sec: 83928.96766386919, Loss: 2.280137062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5363 , TFLOPS: 102.72370768425462, Tokens per sec: 83938.18589228764, Loss: 2.2870824337005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5364 , TFLOPS: 101.55274099342914, Tokens per sec: 82981.35886584988, Loss: 2.2788562774658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5365 , TFLOPS: 102.80021496359768, Tokens per sec: 84000.70195971143, Loss: 2.294900894165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5366 , TFLOPS: 101.36206979790992, Tokens per sec: 82825.55652367791, Loss: 2.2886428833007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5367 , TFLOPS: 101.58726938765261, Tokens per sec: 83009.57290560982, Loss: 2.2882001399993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5368 , TFLOPS: 101.28319345862488, Tokens per sec: 82761.10463639049, Loss: 2.2491540908813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5369 , TFLOPS: 101.46476172802814, Tokens per sec: 82909.46874330295, Loss: 2.2790369987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5370 , TFLOPS: 101.39317646920081, Tokens per sec: 82850.97458554665, Loss: 2.276212692260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5371 , TFLOPS: 102.04271378156044, Tokens per sec: 83381.72824405362, Loss: 2.2815394401550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5372 , TFLOPS: 100.85945170455705, Tokens per sec: 82414.85434105851, Loss: 2.2869420051574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5373 , TFLOPS: 102.12314776773503, Tokens per sec: 83447.45292471189, Loss: 2.287614583969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5374 , TFLOPS: 100.24723624051973, Tokens per sec: 81914.59732556553, Loss: 2.285938024520874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5375 , TFLOPS: 100.32028269738538, Tokens per sec: 81974.28546585354, Loss: 2.283109426498413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5376 , TFLOPS: 102.10124524301305, Tokens per sec: 83429.55580794041, Loss: 2.2971901893615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5377 , TFLOPS: 101.06882835959863, Tokens per sec: 82585.94139572757, Loss: 2.291879653930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5378 , TFLOPS: 101.02250396203418, Tokens per sec: 82548.08853798157, Loss: 2.2998647689819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5379 , TFLOPS: 101.63299682424962, Tokens per sec: 83046.93797118217, Loss: 2.2990269660949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5380 , TFLOPS: 100.96058215432194, Tokens per sec: 82497.49063489077, Loss: 2.3200948238372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5381 , TFLOPS: 102.6781451474465, Tokens per sec: 83900.95557057767, Loss: 2.3084702491760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5382 , TFLOPS: 101.73015784093593, Tokens per sec: 83126.33073906363, Loss: 2.291182041168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5383 , TFLOPS: 102.25878234012761, Tokens per sec: 83558.28342535837, Loss: 2.2965121269226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5384 , TFLOPS: 100.16458399931352, Tokens per sec: 81847.06005161797, Loss: 2.3014988899230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5385 , TFLOPS: 102.72869851973404, Tokens per sec: 83942.2640324334, Loss: 2.314729928970337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5386 , TFLOPS: 102.20513879934028, Tokens per sec: 83514.44990727346, Loss: 2.3007562160491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5387 , TFLOPS: 102.06954862958818, Tokens per sec: 83403.65568916753, Loss: 2.293898105621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5388 , TFLOPS: 101.52025775066386, Tokens per sec: 82954.81597199332, Loss: 2.2949907779693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5389 , TFLOPS: 102.39281514765915, Tokens per sec: 83667.80508270381, Loss: 2.2768924236297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5390 , TFLOPS: 103.34932515016605, Tokens per sec: 84449.39402851017, Loss: 2.3067798614501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5391 , TFLOPS: 101.50975562617997, Tokens per sec: 82946.2344157288, Loss: 2.281568765640259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5392 , TFLOPS: 102.14773861811935, Tokens per sec: 83467.54674158571, Loss: 2.283745765686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5393 , TFLOPS: 101.36746681707898, Tokens per sec: 82829.96656697257, Loss: 2.2960386276245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5394 , TFLOPS: 102.70871840304851, Tokens per sec: 83925.9377647557, Loss: 2.2741963863372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5395 , TFLOPS: 102.03124778754828, Tokens per sec: 83372.35908518535, Loss: 2.301588535308838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5396 , TFLOPS: 101.71441353417833, Tokens per sec: 83113.46566071748, Loss: 2.2734882831573486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5397 , TFLOPS: 102.8613065074201, Tokens per sec: 84050.62143280497, Loss: 2.3068454265594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5398 , TFLOPS: 102.68695516106916, Tokens per sec: 83908.15445949869, Loss: 2.301273822784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5399 , TFLOPS: 99.95856490836653, Tokens per sec: 81678.71654900197, Loss: 2.291454553604126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5400 , TFLOPS: 102.54446089923296, Tokens per sec: 83791.71873002368, Loss: 2.2983341217041016 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/optimizer.pt +[rank0]:[2024-08-27 22:05:16,861] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007668020989513025, 'preprocessing_with_comm': 0.0015394919901154935, 'state_converting': 3.3313983169791754, : 3.342254881019471}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400, took 15.96s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003400 + eval ppl=7.894956588745117, eval loss=2.0662240982055664 +------------------------------------------------------------------ +iteration: 5401 , TFLOPS: 101.48942835411503, Tokens per sec: 82929.624478454, Loss: 2.287187099456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5402 , TFLOPS: 100.5071226414978, Tokens per sec: 82126.95719387585, Loss: 2.2951242923736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5403 , TFLOPS: 100.4794882170159, Tokens per sec: 82104.3763942583, Loss: 2.310091257095337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5404 , TFLOPS: 102.17810344177816, Tokens per sec: 83492.35862065743, Loss: 2.2725603580474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5405 , TFLOPS: 100.43186789184081, Tokens per sec: 82065.46460070151, Loss: 2.2785415649414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5406 , TFLOPS: 103.32648305961823, Tokens per sec: 84430.72916831583, Loss: 2.292130947113037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5407 , TFLOPS: 102.34805744291796, Tokens per sec: 83631.23240999396, Loss: 2.282557964324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5408 , TFLOPS: 102.75030311573941, Tokens per sec: 83959.91770398118, Loss: 2.2944724559783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5409 , TFLOPS: 101.9702970760529, Tokens per sec: 83322.55468981157, Loss: 2.3011515140533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5410 , TFLOPS: 103.39772529582362, Tokens per sec: 84488.94303345756, Loss: 2.2888615131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5411 , TFLOPS: 102.51683152282448, Tokens per sec: 83769.1420553169, Loss: 2.3027350902557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5412 , TFLOPS: 103.41942848143302, Tokens per sec: 84506.67726510868, Loss: 2.28987455368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5413 , TFLOPS: 100.65328311997877, Tokens per sec: 82246.38868334808, Loss: 2.2886714935302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5414 , TFLOPS: 102.36063598434336, Tokens per sec: 83641.51065999312, Loss: 2.2952916622161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5415 , TFLOPS: 102.58493000906653, Tokens per sec: 83824.78708143627, Loss: 2.2976982593536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5416 , TFLOPS: 102.79054840875338, Tokens per sec: 83992.80316890888, Loss: 2.315904140472412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5417 , TFLOPS: 100.19895167136627, Tokens per sec: 81875.14276115484, Loss: 2.2864830493927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5418 , TFLOPS: 103.28343584462887, Tokens per sec: 84395.5541808141, Loss: 2.312095880508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5419 , TFLOPS: 103.35485072749468, Tokens per sec: 84453.90911998639, Loss: 2.3070523738861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5420 , TFLOPS: 100.77047137479022, Tokens per sec: 82342.14622304855, Loss: 2.2680442333221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5421 , TFLOPS: 101.54644839788415, Tokens per sec: 82976.21702404425, Loss: 2.2941629886627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5422 , TFLOPS: 101.42839993118, Tokens per sec: 82879.75658306241, Loss: 2.288862943649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5423 , TFLOPS: 102.68855577039109, Tokens per sec: 83909.46235857898, Loss: 2.2889795303344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5424 , TFLOPS: 101.57143426467916, Tokens per sec: 82996.63361899588, Loss: 2.295969247817993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5425 , TFLOPS: 101.45114812787318, Tokens per sec: 82898.3447201711, Loss: 2.2986032962799072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5426 , TFLOPS: 101.4532388005067, Tokens per sec: 82900.05306259864, Loss: 2.2867627143859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5427 , TFLOPS: 101.5304476057934, Tokens per sec: 82963.14236492987, Loss: 2.3017163276672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5428 , TFLOPS: 100.47513926757638, Tokens per sec: 82100.82275571936, Loss: 2.306063413619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5429 , TFLOPS: 101.96642561153287, Tokens per sec: 83319.39121649189, Loss: 2.2954416275024414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5430 , TFLOPS: 100.46386251607396, Tokens per sec: 82091.60823177731, Loss: 2.3218698501586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5431 , TFLOPS: 102.26585604548184, Tokens per sec: 83564.06353209668, Loss: 2.3007030487060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5432 , TFLOPS: 100.36203229376825, Tokens per sec: 82008.4001357882, Loss: 2.2961714267730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5433 , TFLOPS: 101.6647486530826, Tokens per sec: 83072.88320788638, Loss: 2.294327735900879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5434 , TFLOPS: 100.69948177043271, Tokens per sec: 82284.13878989306, Loss: 2.2819550037384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5435 , TFLOPS: 100.99961477022421, Tokens per sec: 82529.38519013343, Loss: 2.2991087436676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5436 , TFLOPS: 102.20881809661931, Tokens per sec: 83517.45635579368, Loss: 2.26743483543396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5437 , TFLOPS: 101.12874595082607, Tokens per sec: 82634.90160193553, Loss: 2.3198354244232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5438 , TFLOPS: 102.18356429503955, Tokens per sec: 83496.82082443217, Loss: 2.2931644916534424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5439 , TFLOPS: 102.08154614321414, Tokens per sec: 83413.45916639476, Loss: 2.2929155826568604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5440 , TFLOPS: 101.59271853202907, Tokens per sec: 83014.02554175328, Loss: 2.291611909866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5441 , TFLOPS: 101.29359051843552, Tokens per sec: 82769.60034160588, Loss: 2.2622265815734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5442 , TFLOPS: 102.27570183451476, Tokens per sec: 83572.10877976863, Loss: 2.280604839324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5443 , TFLOPS: 101.60035845605319, Tokens per sec: 83020.26832034254, Loss: 2.2994837760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5444 , TFLOPS: 103.3567298251045, Tokens per sec: 84455.44457901566, Loss: 2.2853057384490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5445 , TFLOPS: 100.48095420767685, Tokens per sec: 82105.5742929653, Loss: 2.3169009685516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5446 , TFLOPS: 102.7338544875372, Tokens per sec: 83946.47710645177, Loss: 2.281074285507202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5447 , TFLOPS: 102.0250502408759, Tokens per sec: 83367.29491026029, Loss: 2.2854843139648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5448 , TFLOPS: 103.44882367454629, Tokens per sec: 84530.69683410114, Loss: 2.2827818393707275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5449 , TFLOPS: 102.5574312257778, Tokens per sec: 83802.3171177294, Loss: 2.3022100925445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5450 , TFLOPS: 103.36562806966634, Tokens per sec: 84462.71556370833, Loss: 2.2938685417175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5451 , TFLOPS: 100.85230335878198, Tokens per sec: 82409.0132437109, Loss: 2.2849009037017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5452 , TFLOPS: 103.46198123344772, Tokens per sec: 84541.44821418518, Loss: 2.2849502563476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5453 , TFLOPS: 102.74024600119579, Tokens per sec: 83951.69977679441, Loss: 2.3019959926605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5454 , TFLOPS: 102.70610530525951, Tokens per sec: 83923.80253528517, Loss: 2.278634548187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5455 , TFLOPS: 99.58914693003067, Tokens per sec: 81376.85560923898, Loss: 2.28153395652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5456 , TFLOPS: 103.33409292199794, Tokens per sec: 84436.94738276156, Loss: 2.2896063327789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5457 , TFLOPS: 103.21331787168786, Tokens per sec: 84338.25898012682, Loss: 2.2812273502349854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5458 , TFLOPS: 101.65418554837508, Tokens per sec: 83064.25182311138, Loss: 2.305140495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5459 , TFLOPS: 101.50268502848397, Tokens per sec: 82940.45684833801, Loss: 2.280183792114258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5460 , TFLOPS: 101.35650593743229, Tokens per sec: 82821.01015005511, Loss: 2.2831647396087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5461 , TFLOPS: 102.73611991373136, Tokens per sec: 83948.32824451229, Loss: 2.272050380706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5462 , TFLOPS: 100.69396281477698, Tokens per sec: 82279.6291091562, Loss: 2.2824199199676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5463 , TFLOPS: 102.23406062421267, Tokens per sec: 83538.08267489068, Loss: 2.308598279953003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5464 , TFLOPS: 101.39862204900925, Tokens per sec: 82855.42430898981, Loss: 2.3098297119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5465 , TFLOPS: 101.4665444355484, Tokens per sec: 82910.92543950873, Loss: 2.2836852073669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5466 , TFLOPS: 99.34315979730047, Tokens per sec: 81175.853190813, Loss: 2.2627925872802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5467 , TFLOPS: 102.7758044388898, Tokens per sec: 83980.75549158982, Loss: 2.2898013591766357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5468 , TFLOPS: 100.5282958510128, Tokens per sec: 82144.25836841745, Loss: 2.267066478729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5469 , TFLOPS: 102.11022783868488, Tokens per sec: 83436.89571809645, Loss: 2.300658941268921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5470 , TFLOPS: 100.1183012679934, Tokens per sec: 81809.2412404328, Loss: 2.287137508392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5471 , TFLOPS: 101.60001200334477, Tokens per sec: 83019.98522491599, Loss: 2.2919394969940186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5472 , TFLOPS: 100.86991671760542, Tokens per sec: 82423.40557261373, Loss: 2.2785375118255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5473 , TFLOPS: 101.05616663332765, Tokens per sec: 82575.59517324972, Loss: 2.304825782775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5474 , TFLOPS: 102.12307385713778, Tokens per sec: 83447.39253046025, Loss: 2.290466547012329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5475 , TFLOPS: 101.66935200699514, Tokens per sec: 83076.6447268692, Loss: 2.2862648963928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5476 , TFLOPS: 101.38025634374614, Tokens per sec: 82840.41721844401, Loss: 2.2914037704467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5477 , TFLOPS: 102.85721183288832, Tokens per sec: 84047.27556883876, Loss: 2.285494089126587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5478 , TFLOPS: 100.88817497778061, Tokens per sec: 82438.3248670123, Loss: 2.2806217670440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5479 , TFLOPS: 101.9830332478869, Tokens per sec: 83332.96174367536, Loss: 2.2879114151000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5480 , TFLOPS: 102.1310125436315, Tokens per sec: 83453.87943555423, Loss: 2.3005590438842773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5481 , TFLOPS: 101.6435952777335, Tokens per sec: 83055.59824035232, Loss: 2.3119330406188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5482 , TFLOPS: 103.31014433256667, Tokens per sec: 84417.37837384573, Loss: 2.2977471351623535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5483 , TFLOPS: 100.2519587668724, Tokens per sec: 81918.45622341694, Loss: 2.3073694705963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5484 , TFLOPS: 99.54179133050408, Tokens per sec: 81338.16012982422, Loss: 2.306067943572998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5485 , TFLOPS: 102.02968801334731, Tokens per sec: 83371.08455353357, Loss: 2.308117389678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5486 , TFLOPS: 102.70828347339824, Tokens per sec: 83925.58237254238, Loss: 2.295488119125366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5487 , TFLOPS: 102.54262547731476, Tokens per sec: 83790.21895952636, Loss: 2.2756850719451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5488 , TFLOPS: 102.72955110772212, Tokens per sec: 83942.96070402603, Loss: 2.2874910831451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5489 , TFLOPS: 100.80246884705927, Tokens per sec: 82368.29218132782, Loss: 2.2937774658203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5490 , TFLOPS: 103.42522460126801, Tokens per sec: 84511.41342383127, Loss: 2.3199477195739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5491 , TFLOPS: 102.63284902809819, Tokens per sec: 83863.94294543234, Loss: 2.2701025009155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5492 , TFLOPS: 102.63550097870896, Tokens per sec: 83866.10992254368, Loss: 2.26659893989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5493 , TFLOPS: 100.2515639181784, Tokens per sec: 81918.13358238476, Loss: 2.288799285888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5494 , TFLOPS: 103.36451196292701, Tokens per sec: 84461.80356416048, Loss: 2.2898147106170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5495 , TFLOPS: 103.4322636712836, Tokens per sec: 84517.16522914277, Loss: 2.2816669940948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5496 , TFLOPS: 100.7497422427092, Tokens per sec: 82325.20791560983, Loss: 2.291172504425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5497 , TFLOPS: 101.36559360179015, Tokens per sec: 82828.43591453922, Loss: 2.285646438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5498 , TFLOPS: 101.47064582381716, Tokens per sec: 82914.27678944227, Loss: 2.2978515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5499 , TFLOPS: 102.62531438183882, Tokens per sec: 83857.78619201474, Loss: 2.2946181297302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5500 , TFLOPS: 101.46805258557268, Tokens per sec: 82912.1577877168, Loss: 2.306168556213379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5501 , TFLOPS: 101.49403807471346, Tokens per sec: 82933.39119981974, Loss: 2.3023223876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5502 , TFLOPS: 101.01223129142267, Tokens per sec: 82539.69447438284, Loss: 2.2957797050476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5503 , TFLOPS: 101.75997716116149, Tokens per sec: 83150.69687323752, Loss: 2.281618595123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5504 , TFLOPS: 100.19758419353762, Tokens per sec: 81874.02535981905, Loss: 2.28243350982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5505 , TFLOPS: 101.62966582251708, Tokens per sec: 83044.2161239189, Loss: 2.2945098876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5506 , TFLOPS: 100.0279208506929, Tokens per sec: 81735.38907485762, Loss: 2.3021180629730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5507 , TFLOPS: 102.09648582767083, Tokens per sec: 83425.66676714603, Loss: 2.2790157794952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5508 , TFLOPS: 100.24470602050263, Tokens per sec: 81912.5298176559, Loss: 2.3066492080688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5509 , TFLOPS: 101.65790555154477, Tokens per sec: 83067.29153346273, Loss: 2.29205322265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5510 , TFLOPS: 100.91444521602635, Tokens per sec: 82459.79095493899, Loss: 2.2827606201171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5511 , TFLOPS: 101.08356671938289, Tokens per sec: 82597.98448890641, Loss: 2.2760047912597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5512 , TFLOPS: 102.27118756873028, Tokens per sec: 83568.4200569886, Loss: 2.2668159008026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5513 , TFLOPS: 100.91969947334388, Tokens per sec: 82464.08434384977, Loss: 2.3012542724609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5514 , TFLOPS: 102.18950106706176, Tokens per sec: 83501.67191367762, Loss: 2.2700960636138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5515 , TFLOPS: 102.1263230145464, Tokens per sec: 83450.04750061956, Loss: 2.298923969268799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5516 , TFLOPS: 101.6210900494629, Tokens per sec: 83037.2086389963, Loss: 2.3078420162200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5517 , TFLOPS: 101.1797704139281, Tokens per sec: 82676.59500422273, Loss: 2.292027473449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5518 , TFLOPS: 102.04944052985472, Tokens per sec: 83387.22484324679, Loss: 2.2783169746398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5519 , TFLOPS: 102.23019896909278, Tokens per sec: 83534.92721708439, Loss: 2.293684720993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5520 , TFLOPS: 103.25118388238577, Tokens per sec: 84369.2002722259, Loss: 2.3040759563446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5521 , TFLOPS: 100.28169296930749, Tokens per sec: 81942.7528056531, Loss: 2.291635274887085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5522 , TFLOPS: 102.64646876305311, Tokens per sec: 83875.07198146716, Loss: 2.2850756645202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5523 , TFLOPS: 101.29618215112704, Tokens per sec: 82771.7180313927, Loss: 2.2819502353668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5524 , TFLOPS: 103.24660857660037, Tokens per sec: 84365.4616720898, Loss: 2.289992570877075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5525 , TFLOPS: 102.03983007251027, Tokens per sec: 83379.37189115545, Loss: 2.2718119621276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5526 , TFLOPS: 103.4346929963972, Tokens per sec: 84519.1502932295, Loss: 2.3015389442443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5527 , TFLOPS: 101.5681912778166, Tokens per sec: 82993.98369094857, Loss: 2.3125901222229004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5528 , TFLOPS: 103.43461832787361, Tokens per sec: 84519.08927965672, Loss: 2.282972574234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5529 , TFLOPS: 102.12750156154657, Tokens per sec: 83451.01052171191, Loss: 2.290426731109619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5530 , TFLOPS: 102.6646937461402, Tokens per sec: 83889.96408430042, Loss: 2.298386812210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5531 , TFLOPS: 101.00628226299465, Tokens per sec: 82534.8333701127, Loss: 2.2896459102630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5532 , TFLOPS: 101.9731846603344, Tokens per sec: 83324.91420926101, Loss: 2.304783582687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5533 , TFLOPS: 103.24506201065705, Tokens per sec: 84364.19793324538, Loss: 2.316301107406616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5534 , TFLOPS: 101.4679519256349, Tokens per sec: 82912.07553589043, Loss: 2.305030107498169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5535 , TFLOPS: 101.56151630419284, Tokens per sec: 82988.5293981715, Loss: 2.2981112003326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5536 , TFLOPS: 101.07801091002139, Tokens per sec: 82593.44469405788, Loss: 2.2873940467834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5537 , TFLOPS: 103.23453485830802, Tokens per sec: 84355.59592606798, Loss: 2.288492441177368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5538 , TFLOPS: 100.24626321842484, Tokens per sec: 81913.80224416398, Loss: 2.272146224975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5539 , TFLOPS: 102.18022437891199, Tokens per sec: 83494.09169298726, Loss: 2.2944436073303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5540 , TFLOPS: 101.37462009397687, Tokens per sec: 82835.81169366757, Loss: 2.3174009323120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5541 , TFLOPS: 101.4771359702708, Tokens per sec: 82919.58005518076, Loss: 2.285907506942749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5542 , TFLOPS: 100.289760577442, Tokens per sec: 81949.34505593852, Loss: 2.2792279720306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5543 , TFLOPS: 101.49781839891297, Tokens per sec: 82936.48020003736, Loss: 2.273465156555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5544 , TFLOPS: 101.10458504587355, Tokens per sec: 82615.15910454179, Loss: 2.278557777404785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5545 , TFLOPS: 101.75473104597063, Tokens per sec: 83146.41013746808, Loss: 2.273355722427368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5546 , TFLOPS: 100.86685068456947, Tokens per sec: 82420.90023809334, Loss: 2.2893688678741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5547 , TFLOPS: 101.18751334232292, Tokens per sec: 82682.92195033483, Loss: 2.291973829269409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5548 , TFLOPS: 101.61940873321495, Tokens per sec: 83035.83479220916, Loss: 2.2633538246154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5549 , TFLOPS: 102.32669063500452, Tokens per sec: 83613.77304121733, Loss: 2.2968082427978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5550 , TFLOPS: 101.4777946994769, Tokens per sec: 82920.11831977226, Loss: 2.300590753555298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5551 , TFLOPS: 102.1477675559273, Tokens per sec: 83467.57038741352, Loss: 2.2709267139434814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5552 , TFLOPS: 101.4737454932421, Tokens per sec: 82916.80960912196, Loss: 2.2868494987487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5553 , TFLOPS: 102.18107950717318, Tokens per sec: 83494.79044030244, Loss: 2.2880747318267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5554 , TFLOPS: 101.53100990177464, Tokens per sec: 82963.60183145091, Loss: 2.2959487438201904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5555 , TFLOPS: 101.34710466291851, Tokens per sec: 82813.32812663956, Loss: 2.274898052215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5556 , TFLOPS: 102.11365193604851, Tokens per sec: 83439.69363619637, Loss: 2.2690160274505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5557 , TFLOPS: 100.92644566017005, Tokens per sec: 82469.59682676772, Loss: 2.2969255447387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5558 , TFLOPS: 103.37891353557137, Tokens per sec: 84473.57145989765, Loss: 2.2963004112243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5559 , TFLOPS: 101.03863883842511, Tokens per sec: 82561.27276082928, Loss: 2.2987115383148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5560 , TFLOPS: 102.7045428563563, Tokens per sec: 83922.52581806536, Loss: 2.2784817218780518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5561 , TFLOPS: 100.8642745806196, Tokens per sec: 82418.79523724108, Loss: 2.2783775329589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5562 , TFLOPS: 102.7885360141914, Tokens per sec: 83991.15878950866, Loss: 2.2930431365966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5563 , TFLOPS: 102.68131619234131, Tokens per sec: 83903.54671299118, Loss: 2.3093655109405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5564 , TFLOPS: 101.93623980182417, Tokens per sec: 83294.72561432705, Loss: 2.29194712638855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5565 , TFLOPS: 101.46742055511045, Tokens per sec: 82911.64133935621, Loss: 2.288639545440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5566 , TFLOPS: 102.0262002279527, Tokens per sec: 83368.23459430406, Loss: 2.287865400314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5567 , TFLOPS: 103.45320865786246, Tokens per sec: 84534.27991684829, Loss: 2.2719650268554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5568 , TFLOPS: 102.05674505626853, Tokens per sec: 83393.19355981474, Loss: 2.2912850379943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5569 , TFLOPS: 100.80458811247142, Tokens per sec: 82370.02388765084, Loss: 2.2977101802825928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5570 , TFLOPS: 102.71467169371579, Tokens per sec: 83930.80235181126, Loss: 2.2937064170837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5571 , TFLOPS: 102.76434535856328, Tokens per sec: 83971.39198207203, Loss: 2.3132951259613037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5572 , TFLOPS: 101.52166226352728, Tokens per sec: 82955.96363560938, Loss: 2.281144857406616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5573 , TFLOPS: 101.4198859439085, Tokens per sec: 82872.79958489334, Loss: 2.2980496883392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5574 , TFLOPS: 100.27678159748973, Tokens per sec: 81938.73959731087, Loss: 2.2694075107574463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5575 , TFLOPS: 102.86631018869403, Tokens per sec: 84054.7100695801, Loss: 2.2913215160369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5576 , TFLOPS: 101.59714733993744, Tokens per sec: 83017.64443470286, Loss: 2.3016440868377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5577 , TFLOPS: 101.4230595460393, Tokens per sec: 82875.3928168902, Loss: 2.283214807510376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5578 , TFLOPS: 101.40957201823495, Tokens per sec: 82864.37181072142, Loss: 2.2847936153411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5579 , TFLOPS: 101.5645983546322, Tokens per sec: 82991.04782092453, Loss: 2.3299577236175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5580 , TFLOPS: 100.88290939855335, Tokens per sec: 82434.02222667764, Loss: 2.2846665382385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5581 , TFLOPS: 101.91693469441094, Tokens per sec: 83278.95091410202, Loss: 2.301769495010376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5582 , TFLOPS: 100.53963681326506, Tokens per sec: 82153.5253606161, Loss: 2.2910513877868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5583 , TFLOPS: 102.89032742176951, Tokens per sec: 84074.33516898466, Loss: 2.294562339782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5584 , TFLOPS: 101.07402911167848, Tokens per sec: 82590.19106413129, Loss: 2.2749276161193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5585 , TFLOPS: 100.3363096939955, Tokens per sec: 81987.38153735525, Loss: 2.3149452209472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5586 , TFLOPS: 101.57322676302319, Tokens per sec: 82998.09831553581, Loss: 2.2945070266723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5587 , TFLOPS: 101.08567501223018, Tokens per sec: 82599.70722926424, Loss: 2.288788318634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5588 , TFLOPS: 102.74058087369009, Tokens per sec: 83951.9734097299, Loss: 2.294830083847046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5589 , TFLOPS: 101.71960725203571, Tokens per sec: 83117.70958127695, Loss: 2.3017451763153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5590 , TFLOPS: 102.13957659931114, Tokens per sec: 83460.87734590846, Loss: 2.3003406524658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5591 , TFLOPS: 102.26134234910565, Tokens per sec: 83560.37527459483, Loss: 2.2967660427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5592 , TFLOPS: 101.42475483462694, Tokens per sec: 82876.77808083578, Loss: 2.2802658081054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5593 , TFLOPS: 102.00234997346926, Tokens per sec: 83348.74593740547, Loss: 2.2917933464050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5594 , TFLOPS: 102.02561009354814, Tokens per sec: 83367.75238029067, Loss: 2.269834518432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5595 , TFLOPS: 102.14419870700004, Tokens per sec: 83464.65418908471, Loss: 2.2888340950012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5596 , TFLOPS: 102.70125108822019, Tokens per sec: 83919.83602958369, Loss: 2.289097785949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5597 , TFLOPS: 101.49877291885824, Tokens per sec: 82937.26016285617, Loss: 2.2851147651672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5598 , TFLOPS: 102.80861639888698, Tokens per sec: 84007.5669887585, Loss: 2.297597646713257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5599 , TFLOPS: 100.31486102570291, Tokens per sec: 81969.85527835555, Loss: 2.294811964035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5600 , TFLOPS: 103.32021981027403, Tokens per sec: 84425.6113060463, Loss: 2.29850697517395 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/optimizer.pt +[rank0]:[2024-08-27 22:58:12,372] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007660259987460449, 'preprocessing_with_comm': 0.001544163009384647, 'state_converting': 3.3183553330018185, : 3.3291619239898864}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600, took 15.86s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003600 + eval ppl=7.791993618011475, eval loss=2.0530967712402344 +------------------------------------------------------------------ +iteration: 5601 , TFLOPS: 101.2395572719772, Tokens per sec: 82725.4483849853, Loss: 2.2828497886657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5602 , TFLOPS: 100.8426411983468, Tokens per sec: 82401.11804369318, Loss: 2.2918665409088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5603 , TFLOPS: 100.37175500936497, Tokens per sec: 82016.34481698724, Loss: 2.269803047180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5604 , TFLOPS: 101.06863708205972, Tokens per sec: 82585.78509792661, Loss: 2.2974953651428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5605 , TFLOPS: 100.61690205303431, Tokens per sec: 82216.66077701577, Loss: 2.2924108505249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5606 , TFLOPS: 102.12914604252322, Tokens per sec: 83452.35426945065, Loss: 2.2798075675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5607 , TFLOPS: 101.59363529796343, Tokens per sec: 83014.7746547979, Loss: 2.2806570529937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5608 , TFLOPS: 102.88666322900103, Tokens per sec: 84071.34106274883, Loss: 2.290902614593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5609 , TFLOPS: 101.90760191270087, Tokens per sec: 83271.32485791954, Loss: 2.3056628704071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5610 , TFLOPS: 103.29250642842712, Tokens per sec: 84402.96598833326, Loss: 2.2815163135528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5611 , TFLOPS: 102.78090427772663, Tokens per sec: 83984.92270118493, Loss: 2.2911462783813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5612 , TFLOPS: 103.42636301859389, Tokens per sec: 84512.3436539337, Loss: 2.300677537918091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5613 , TFLOPS: 101.46795636929316, Tokens per sec: 82912.07916691799, Loss: 2.285667896270752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5614 , TFLOPS: 101.02286978295275, Tokens per sec: 82548.38745967121, Loss: 2.28261137008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5615 , TFLOPS: 101.90444502514342, Tokens per sec: 83268.74528383082, Loss: 2.3079535961151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5616 , TFLOPS: 102.68607785205735, Tokens per sec: 83907.43758772119, Loss: 2.2913458347320557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5617 , TFLOPS: 102.77137527688858, Tokens per sec: 83977.13630930184, Loss: 2.2918686866760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5618 , TFLOPS: 103.36818768535055, Tokens per sec: 84464.80709157427, Loss: 2.270160436630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5619 , TFLOPS: 102.0682200586397, Tokens per sec: 83402.57007964507, Loss: 2.29229998588562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5620 , TFLOPS: 102.1207661034633, Tokens per sec: 83445.50680552596, Loss: 2.2963662147521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5621 , TFLOPS: 101.45994513250372, Tokens per sec: 82905.53297910975, Loss: 2.2929747104644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5622 , TFLOPS: 103.35142592235049, Tokens per sec: 84451.11062354075, Loss: 2.294715642929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5623 , TFLOPS: 100.73536060483788, Tokens per sec: 82313.45630908887, Loss: 2.285409450531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5624 , TFLOPS: 102.23826004252291, Tokens per sec: 83541.51412769464, Loss: 2.256420135498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5625 , TFLOPS: 101.50600186543967, Tokens per sec: 82943.16712119749, Loss: 2.297563076019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5626 , TFLOPS: 102.24049997918395, Tokens per sec: 83543.34443760147, Loss: 2.2903223037719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5627 , TFLOPS: 102.13933079479806, Tokens per sec: 83460.67649271325, Loss: 2.280489444732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5628 , TFLOPS: 101.4752489770324, Tokens per sec: 82918.03814443013, Loss: 2.2835752964019775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5629 , TFLOPS: 101.04600211923072, Tokens per sec: 82567.28948712324, Loss: 2.287247657775879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5630 , TFLOPS: 102.18713291046183, Tokens per sec: 83499.73683195806, Loss: 2.310175895690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5631 , TFLOPS: 101.88097247154901, Tokens per sec: 83249.56525605156, Loss: 2.3182835578918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5632 , TFLOPS: 99.98454469140273, Tokens per sec: 81699.94529849997, Loss: 2.2970168590545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5633 , TFLOPS: 102.11889139742082, Tokens per sec: 83443.97493495943, Loss: 2.269592523574829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5634 , TFLOPS: 100.34822811285142, Tokens per sec: 81997.12038420982, Loss: 2.2929391860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5635 , TFLOPS: 102.02161423031235, Tokens per sec: 83364.48725757803, Loss: 2.299710750579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5636 , TFLOPS: 100.70750948794296, Tokens per sec: 82290.69844452251, Loss: 2.303835153579712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5637 , TFLOPS: 102.25504065966457, Tokens per sec: 83555.2260019326, Loss: 2.2874677181243896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5638 , TFLOPS: 100.15015996629647, Tokens per sec: 81835.2738029321, Loss: 2.293295383453369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5639 , TFLOPS: 101.78634866018248, Tokens per sec: 83172.24570395036, Loss: 2.2634243965148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5640 , TFLOPS: 100.31115961975054, Tokens per sec: 81966.8307642692, Loss: 2.2799293994903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5641 , TFLOPS: 101.28568459279263, Tokens per sec: 82763.14020624645, Loss: 2.2994325160980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5642 , TFLOPS: 100.36637801982087, Tokens per sec: 82011.95114041475, Loss: 2.2940518856048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5643 , TFLOPS: 101.62228031840195, Tokens per sec: 83038.18123838663, Loss: 2.292746067047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5644 , TFLOPS: 101.48504069026171, Tokens per sec: 82926.03920537094, Loss: 2.298933982849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5645 , TFLOPS: 102.77986559401569, Tokens per sec: 83984.07396598661, Loss: 2.2895596027374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5646 , TFLOPS: 102.69111036266332, Tokens per sec: 83911.54978168567, Loss: 2.272930145263672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5647 , TFLOPS: 102.57305895144111, Tokens per sec: 83815.08693466234, Loss: 2.3127026557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5648 , TFLOPS: 102.18812382652543, Tokens per sec: 83500.54653498207, Loss: 2.29642653465271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5649 , TFLOPS: 102.75412472129373, Tokens per sec: 83963.04043625649, Loss: 2.295135259628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5650 , TFLOPS: 103.32979277356453, Tokens per sec: 84433.43362078095, Loss: 2.286240577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5651 , TFLOPS: 102.09876601505353, Tokens per sec: 83427.52996695368, Loss: 2.26884388923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5652 , TFLOPS: 102.21776631591058, Tokens per sec: 83524.76817612424, Loss: 2.281024694442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5653 , TFLOPS: 101.49379527126682, Tokens per sec: 82933.19279887318, Loss: 2.276735782623291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5654 , TFLOPS: 103.26645235992336, Tokens per sec: 84381.67653826746, Loss: 2.285191774368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5655 , TFLOPS: 102.7686159304968, Tokens per sec: 83974.88157633322, Loss: 2.277589797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5656 , TFLOPS: 102.73274326659195, Tokens per sec: 83945.56909921205, Loss: 2.2966887950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5657 , TFLOPS: 102.20648511363649, Tokens per sec: 83515.5500153421, Loss: 2.2657082080841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5658 , TFLOPS: 101.44542941278549, Tokens per sec: 82893.67181086012, Loss: 2.2943999767303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5659 , TFLOPS: 101.4487975954244, Tokens per sec: 82896.4240395991, Loss: 2.2657015323638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5660 , TFLOPS: 103.41786690853557, Tokens per sec: 84505.40126369572, Loss: 2.2924134731292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5661 , TFLOPS: 101.5699863729771, Tokens per sec: 82995.45050941408, Loss: 2.277080535888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5662 , TFLOPS: 102.2226617327764, Tokens per sec: 83528.76834726454, Loss: 2.272392749786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5663 , TFLOPS: 101.42617437667742, Tokens per sec: 82877.93802518748, Loss: 2.311088800430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5664 , TFLOPS: 101.62424197990505, Tokens per sec: 83039.78416249849, Loss: 2.278862714767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5665 , TFLOPS: 101.8751787929404, Tokens per sec: 83244.8310921179, Loss: 2.297469139099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5666 , TFLOPS: 101.26344607887043, Tokens per sec: 82744.96854405035, Loss: 2.269094705581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5667 , TFLOPS: 100.91055169431799, Tokens per sec: 82456.60945812293, Loss: 2.2833173274993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5668 , TFLOPS: 102.0836727440901, Tokens per sec: 83415.19686671409, Loss: 2.2806754112243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5669 , TFLOPS: 100.96670956194137, Tokens per sec: 82502.49749739011, Loss: 2.2782769203186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5670 , TFLOPS: 101.12877467713253, Tokens per sec: 82634.92507494001, Loss: 2.3010144233703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5671 , TFLOPS: 101.76630945885755, Tokens per sec: 83155.87115669283, Loss: 2.281873941421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5672 , TFLOPS: 100.44568871137459, Tokens per sec: 82076.7579481221, Loss: 2.276641368865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5673 , TFLOPS: 101.96348512689849, Tokens per sec: 83316.98847080233, Loss: 2.2686102390289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5674 , TFLOPS: 100.98533295911307, Tokens per sec: 82517.715154628, Loss: 2.2907280921936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5675 , TFLOPS: 102.86623281937078, Tokens per sec: 84054.64684911442, Loss: 2.303370952606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5676 , TFLOPS: 100.0278706368597, Tokens per sec: 81735.3480438419, Loss: 2.323915958404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5677 , TFLOPS: 102.17688762623645, Tokens per sec: 83491.36514648043, Loss: 2.302489995956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5678 , TFLOPS: 101.45157961691514, Tokens per sec: 82898.6973009748, Loss: 2.2956862449645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5679 , TFLOPS: 100.60673468540692, Tokens per sec: 82208.3527591961, Loss: 2.3043386936187744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5680 , TFLOPS: 100.33127422152306, Tokens per sec: 81983.26692317343, Loss: 2.288745403289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5681 , TFLOPS: 101.10278623351793, Tokens per sec: 82613.68924866048, Loss: 2.2780258655548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5682 , TFLOPS: 102.19111919415282, Tokens per sec: 83502.99412697804, Loss: 2.2874596118927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5683 , TFLOPS: 102.83279692393168, Tokens per sec: 84027.32551823462, Loss: 2.294098138809204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5684 , TFLOPS: 102.51448377049795, Tokens per sec: 83767.22364645438, Loss: 2.276327133178711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5685 , TFLOPS: 101.99747021808064, Tokens per sec: 83344.75856366134, Loss: 2.2726891040802 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5686 , TFLOPS: 102.63811211246676, Tokens per sec: 83868.24354715372, Loss: 2.2971465587615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5687 , TFLOPS: 102.5833611386052, Tokens per sec: 83823.50511699599, Loss: 2.286656379699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5688 , TFLOPS: 102.7280859945767, Tokens per sec: 83941.76352235954, Loss: 2.2881455421447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5689 , TFLOPS: 101.38234341030203, Tokens per sec: 82842.1226142527, Loss: 2.2737560272216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5690 , TFLOPS: 102.10793505513989, Tokens per sec: 83435.02222564034, Loss: 2.280829668045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5691 , TFLOPS: 100.70729709500606, Tokens per sec: 82290.52489278628, Loss: 2.2957332134246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5692 , TFLOPS: 103.296333465372, Tokens per sec: 84406.09315873763, Loss: 2.2843503952026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5693 , TFLOPS: 102.7345630592906, Tokens per sec: 83947.05609867175, Loss: 2.2793619632720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5694 , TFLOPS: 102.73094038491325, Tokens per sec: 83944.09591818208, Loss: 2.286497116088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5695 , TFLOPS: 102.1919418086469, Tokens per sec: 83503.66630645713, Loss: 2.276526927947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5696 , TFLOPS: 101.56392523808593, Tokens per sec: 82990.49780006701, Loss: 2.279820680618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5697 , TFLOPS: 101.90165532689308, Tokens per sec: 83266.46574957726, Loss: 2.2932257652282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5698 , TFLOPS: 102.71800487362295, Tokens per sec: 83933.5259789171, Loss: 2.261808395385742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5699 , TFLOPS: 101.5468060903939, Tokens per sec: 82976.5093038018, Loss: 2.301645278930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5700 , TFLOPS: 101.40869231164054, Tokens per sec: 82863.65297981746, Loss: 2.281909942626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5701 , TFLOPS: 100.94428260742438, Tokens per sec: 82484.1718555331, Loss: 2.2697770595550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5702 , TFLOPS: 102.17552124612503, Tokens per sec: 83490.24864211761, Loss: 2.299931526184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5703 , TFLOPS: 102.19459716415793, Tokens per sec: 83505.83606580012, Loss: 2.301832675933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5704 , TFLOPS: 100.88677690827025, Tokens per sec: 82437.18246842588, Loss: 2.294046640396118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5705 , TFLOPS: 100.85764634217249, Tokens per sec: 82413.37913297991, Loss: 2.296489953994751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5706 , TFLOPS: 102.6533305967307, Tokens per sec: 83880.67896240443, Loss: 2.2751448154449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5707 , TFLOPS: 102.02852391138602, Tokens per sec: 83370.13333585433, Loss: 2.290893793106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5708 , TFLOPS: 100.1389151331629, Tokens per sec: 81826.08536031088, Loss: 2.272674798965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5709 , TFLOPS: 102.15263969066912, Tokens per sec: 83471.55153413095, Loss: 2.2905757427215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5710 , TFLOPS: 100.93883137382528, Tokens per sec: 82479.71751223205, Loss: 2.285196304321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5711 , TFLOPS: 102.17306403948464, Tokens per sec: 83488.24079531764, Loss: 2.300248146057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5712 , TFLOPS: 100.38692749544761, Tokens per sec: 82028.74264593991, Loss: 2.304077386856079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5713 , TFLOPS: 102.73217293157593, Tokens per sec: 83945.10306378873, Loss: 2.2743918895721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5714 , TFLOPS: 99.78481553246006, Tokens per sec: 81536.74146124195, Loss: 2.278939723968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5715 , TFLOPS: 102.20141119152002, Tokens per sec: 83511.40398296635, Loss: 2.2827963829040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5716 , TFLOPS: 100.73713520491528, Tokens per sec: 82314.90638049439, Loss: 2.3046936988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5717 , TFLOPS: 100.74989433346335, Tokens per sec: 82325.33219288022, Loss: 2.3034281730651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5718 , TFLOPS: 101.12898396004398, Tokens per sec: 82635.09608539438, Loss: 2.2963085174560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5719 , TFLOPS: 101.03929745047414, Tokens per sec: 82561.81092968871, Loss: 2.2950327396392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5720 , TFLOPS: 101.6797374930333, Tokens per sec: 83085.13097485718, Loss: 2.291609287261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5721 , TFLOPS: 102.13662983594732, Tokens per sec: 83458.46946970744, Loss: 2.2932701110839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5722 , TFLOPS: 102.70230465940045, Tokens per sec: 83920.69692971642, Loss: 2.289714813232422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5723 , TFLOPS: 102.03846227807338, Tokens per sec: 83378.2542311109, Loss: 2.2914886474609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5724 , TFLOPS: 102.82925147437696, Tokens per sec: 84024.42844013545, Loss: 2.3019635677337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5725 , TFLOPS: 103.33067712662533, Tokens per sec: 84434.1562484322, Loss: 2.2923872470855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5726 , TFLOPS: 101.72491221530879, Tokens per sec: 83122.04440332929, Loss: 2.3024890422821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5727 , TFLOPS: 101.97733960981623, Tokens per sec: 83328.30932544052, Loss: 2.2877931594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5728 , TFLOPS: 101.5695171950249, Tokens per sec: 82995.06713203162, Loss: 2.298511028289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5729 , TFLOPS: 102.03213619541211, Tokens per sec: 83373.08502612053, Loss: 2.2931325435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5730 , TFLOPS: 102.77236909682014, Tokens per sec: 83977.94838515112, Loss: 2.2893319129943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5731 , TFLOPS: 102.77636965999812, Tokens per sec: 83981.2173483075, Loss: 2.2793385982513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5732 , TFLOPS: 102.62268548136102, Tokens per sec: 83855.63804976041, Loss: 2.3004043102264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5733 , TFLOPS: 101.53852689801147, Tokens per sec: 82969.74416258074, Loss: 2.2939772605895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5734 , TFLOPS: 101.93949893071434, Tokens per sec: 83297.38873244062, Loss: 2.2945234775543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5735 , TFLOPS: 99.13819111065702, Tokens per sec: 81008.36800059324, Loss: 2.2834770679473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5736 , TFLOPS: 102.87131360563217, Tokens per sec: 84058.79849035968, Loss: 2.2472386360168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5737 , TFLOPS: 101.47552615030239, Tokens per sec: 82918.26462984411, Loss: 2.290708541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5738 , TFLOPS: 102.71320695905708, Tokens per sec: 83929.605479416, Loss: 2.286418914794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5739 , TFLOPS: 101.52486575288295, Tokens per sec: 82958.58128922705, Loss: 2.2947161197662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5740 , TFLOPS: 102.18686406262457, Tokens per sec: 83499.51714946947, Loss: 2.290963888168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5741 , TFLOPS: 101.982662104928, Tokens per sec: 83332.65847320958, Loss: 2.290055274963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5742 , TFLOPS: 102.17073577661417, Tokens per sec: 83486.33831179135, Loss: 2.289414882659912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5743 , TFLOPS: 100.93645879672142, Tokens per sec: 82477.77881840491, Loss: 2.3037397861480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5744 , TFLOPS: 102.03001153380818, Tokens per sec: 83371.3489104303, Loss: 2.272761344909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5745 , TFLOPS: 101.97070967872686, Tokens per sec: 83322.89183807779, Loss: 2.281482696533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5746 , TFLOPS: 99.89434829555982, Tokens per sec: 81626.24350159493, Loss: 2.290738821029663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5747 , TFLOPS: 102.15372456714623, Tokens per sec: 83472.43801462762, Loss: 2.300248146057129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5748 , TFLOPS: 101.09530312956905, Tokens per sec: 82607.5746117916, Loss: 2.28733491897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5749 , TFLOPS: 102.15464702119242, Tokens per sec: 83473.19177557484, Loss: 2.286163806915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5750 , TFLOPS: 100.3388629313985, Tokens per sec: 81989.46785336341, Loss: 2.287776470184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5751 , TFLOPS: 102.82614312906092, Tokens per sec: 84021.8885311618, Loss: 2.278764009475708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5752 , TFLOPS: 100.37197649903864, Tokens per sec: 82016.52580190125, Loss: 2.2921314239501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5753 , TFLOPS: 101.50884878045402, Tokens per sec: 82945.49340874182, Loss: 2.2917497158050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5754 , TFLOPS: 101.45298126220153, Tokens per sec: 82899.84262141987, Loss: 2.2867510318756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5755 , TFLOPS: 100.62557193239422, Tokens per sec: 82223.74515862332, Loss: 2.2930595874786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5756 , TFLOPS: 100.35036010052715, Tokens per sec: 81998.86248622191, Loss: 2.271007537841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5757 , TFLOPS: 101.90246635488782, Tokens per sec: 83267.12846142947, Loss: 2.286017894744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5758 , TFLOPS: 101.0641541473316, Tokens per sec: 82582.12197655927, Loss: 2.2696104049682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5759 , TFLOPS: 102.8492671732613, Tokens per sec: 84040.78378294379, Loss: 2.274271011352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5760 , TFLOPS: 102.14885420703257, Tokens per sec: 83468.45831800452, Loss: 2.2999627590179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5761 , TFLOPS: 102.60642327443433, Tokens per sec: 83842.34978185417, Loss: 2.2684216499328613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5762 , TFLOPS: 102.05236772193643, Tokens per sec: 83389.61672724938, Loss: 2.28706693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5763 , TFLOPS: 103.4395534150647, Tokens per sec: 84523.12186644101, Loss: 2.3151140213012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5764 , TFLOPS: 102.00083207400846, Tokens per sec: 83347.50562268183, Loss: 2.2852392196655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5765 , TFLOPS: 101.47121197190758, Tokens per sec: 82914.73940362089, Loss: 2.2455782890319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5766 , TFLOPS: 101.67263933306631, Tokens per sec: 83079.33088562527, Loss: 2.287065267562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5767 , TFLOPS: 100.96505110544204, Tokens per sec: 82501.14232989203, Loss: 2.284355640411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5768 , TFLOPS: 102.86172413152998, Tokens per sec: 84050.96268421574, Loss: 2.2840981483459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5769 , TFLOPS: 102.71477236534693, Tokens per sec: 83930.8846131926, Loss: 2.292595863342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5770 , TFLOPS: 103.40924955717752, Tokens per sec: 84498.35980407069, Loss: 2.2936086654663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5771 , TFLOPS: 102.11580401458787, Tokens per sec: 83441.4521549703, Loss: 2.295945167541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5772 , TFLOPS: 102.25247640787319, Tokens per sec: 83553.13068578414, Loss: 2.3000221252441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5773 , TFLOPS: 101.9606949081924, Tokens per sec: 83314.70851127096, Loss: 2.2886147499084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5774 , TFLOPS: 103.43998036608326, Tokens per sec: 84523.47073910898, Loss: 2.286989450454712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5775 , TFLOPS: 101.49971654914809, Tokens per sec: 82938.0312274572, Loss: 2.280362129211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5776 , TFLOPS: 101.36523953944761, Tokens per sec: 82828.14660108475, Loss: 2.2788376808166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5777 , TFLOPS: 101.60452861920083, Tokens per sec: 83023.67586800002, Loss: 2.308258056640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5778 , TFLOPS: 102.29019964101185, Tokens per sec: 83583.95531066417, Loss: 2.2864363193511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5779 , TFLOPS: 102.78672694916503, Tokens per sec: 83989.68055590613, Loss: 2.276240110397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5780 , TFLOPS: 102.25261367727649, Tokens per sec: 83553.24285214722, Loss: 2.293002128601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5781 , TFLOPS: 99.32278689789281, Tokens per sec: 81159.20597026192, Loss: 2.2869882583618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5782 , TFLOPS: 102.65914123767956, Tokens per sec: 83885.42698670295, Loss: 2.3012053966522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5783 , TFLOPS: 102.13636970662533, Tokens per sec: 83458.25691134213, Loss: 2.282268762588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5784 , TFLOPS: 100.38855970139291, Tokens per sec: 82030.07636343453, Loss: 2.292736291885376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5785 , TFLOPS: 101.50809483762511, Tokens per sec: 82944.8773426481, Loss: 2.291271686553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5786 , TFLOPS: 101.01831779267663, Tokens per sec: 82544.66791123794, Loss: 2.2968485355377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5787 , TFLOPS: 102.20599284844454, Tokens per sec: 83515.14777278171, Loss: 2.297753095626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5788 , TFLOPS: 100.34726029729958, Tokens per sec: 81996.32955720865, Loss: 2.286961555480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5789 , TFLOPS: 102.85563830638202, Tokens per sec: 84045.98979982428, Loss: 2.2852566242218018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5790 , TFLOPS: 100.49945650067204, Tokens per sec: 82120.69299286309, Loss: 2.2774782180786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5791 , TFLOPS: 101.5806336594231, Tokens per sec: 83004.15068125464, Loss: 2.308332920074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5792 , TFLOPS: 100.97571957827195, Tokens per sec: 82509.85981367229, Loss: 2.294818162918091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5793 , TFLOPS: 100.50657600171175, Tokens per sec: 82126.5105204349, Loss: 2.304877758026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5794 , TFLOPS: 100.53266009170189, Tokens per sec: 82147.82450183005, Loss: 2.287545680999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5795 , TFLOPS: 101.1726820419495, Tokens per sec: 82670.80291300808, Loss: 2.259784698486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5796 , TFLOPS: 101.6705607248315, Tokens per sec: 83077.63240132846, Loss: 2.2766971588134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5797 , TFLOPS: 101.49506743330994, Tokens per sec: 82934.23231522697, Loss: 2.2980401515960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5798 , TFLOPS: 103.37496432395452, Tokens per sec: 84470.34445742377, Loss: 2.2900564670562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5799 , TFLOPS: 101.57818269585937, Tokens per sec: 83002.1479358333, Loss: 2.2627499103546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5800 , TFLOPS: 102.7677556625939, Tokens per sec: 83974.17862928449, Loss: 2.299561023712158 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/optimizer.pt +[rank0]:[2024-08-27 23:51:08,355] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007789905008394271, 'preprocessing_with_comm': 0.0015331649919971824, 'state_converting': 3.3393226619809866, : 3.350366867991397}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800, took 15.81s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0003800 + eval ppl=8.14415168762207, eval loss=2.0973000526428223 +------------------------------------------------------------------ +iteration: 5801 , TFLOPS: 100.13033636340575, Tokens per sec: 81819.07542670514, Loss: 2.275599479675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5802 , TFLOPS: 101.51389434499994, Tokens per sec: 82949.61626941669, Loss: 2.28218674659729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5803 , TFLOPS: 99.85855480298112, Tokens per sec: 81596.99571740242, Loss: 2.2717154026031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5804 , TFLOPS: 101.9794613529148, Tokens per sec: 83330.0430563449, Loss: 2.2834770679473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5805 , TFLOPS: 100.41697841698661, Tokens per sec: 82053.29802750896, Loss: 2.292170763015747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5806 , TFLOPS: 101.1163210458067, Tokens per sec: 82624.74889219868, Loss: 2.296403408050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5807 , TFLOPS: 101.54011867643396, Tokens per sec: 82971.04484570578, Loss: 2.2825658321380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5808 , TFLOPS: 101.28320690733062, Tokens per sec: 82761.11562567414, Loss: 2.2525830268859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5809 , TFLOPS: 102.01169538049454, Tokens per sec: 83356.38231005799, Loss: 2.286749839782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5810 , TFLOPS: 103.36223216011098, Tokens per sec: 84459.94067859215, Loss: 2.2866177558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5811 , TFLOPS: 102.67523248418657, Tokens per sec: 83898.57555844936, Loss: 2.310352087020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5812 , TFLOPS: 103.3247027571675, Tokens per sec: 84429.27443735427, Loss: 2.293494939804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5813 , TFLOPS: 101.76299561470933, Tokens per sec: 83153.16332933339, Loss: 2.2866873741149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5814 , TFLOPS: 103.36537355356938, Tokens per sec: 84462.50759205368, Loss: 2.2845046520233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5815 , TFLOPS: 102.75927892085745, Tokens per sec: 83967.2520654009, Loss: 2.2901504039764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5816 , TFLOPS: 101.38228661660435, Tokens per sec: 82842.07620666033, Loss: 2.2901694774627686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5817 , TFLOPS: 101.4426954699575, Tokens per sec: 82891.4378357974, Loss: 2.2956511974334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5818 , TFLOPS: 103.31987421424667, Tokens per sec: 84425.3289106359, Loss: 2.285813331604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5819 , TFLOPS: 102.0220422425952, Tokens per sec: 83364.83699743237, Loss: 2.292388677597046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5820 , TFLOPS: 103.3851928698821, Tokens per sec: 84478.70246560739, Loss: 2.287795305252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5821 , TFLOPS: 103.44816043537526, Tokens per sec: 84530.15488430118, Loss: 2.2849035263061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5822 , TFLOPS: 100.53623520330297, Tokens per sec: 82150.74581755085, Loss: 2.283134937286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5823 , TFLOPS: 102.12991082396287, Tokens per sec: 83452.97919205221, Loss: 2.283168077468872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5824 , TFLOPS: 103.36880104662367, Tokens per sec: 84465.30828485986, Loss: 2.2832086086273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5825 , TFLOPS: 103.30690218603038, Tokens per sec: 84414.72913245065, Loss: 2.255383014678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5826 , TFLOPS: 101.53794598105179, Tokens per sec: 82969.26948037885, Loss: 2.280784845352173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5827 , TFLOPS: 100.92219431350485, Tokens per sec: 82466.12294196818, Loss: 2.284290313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5828 , TFLOPS: 103.25084587871369, Tokens per sec: 84368.92408072446, Loss: 2.2698168754577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5829 , TFLOPS: 102.80017902733725, Tokens per sec: 84000.67259526794, Loss: 2.2930397987365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5830 , TFLOPS: 100.90830747155614, Tokens per sec: 82454.77564593278, Loss: 2.2971534729003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5831 , TFLOPS: 102.59351956537321, Tokens per sec: 83831.80582900932, Loss: 2.255861759185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5832 , TFLOPS: 101.33371312507009, Tokens per sec: 82802.38555633478, Loss: 2.2819509506225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5833 , TFLOPS: 101.65645842417108, Tokens per sec: 83066.10904843331, Loss: 2.2852518558502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5834 , TFLOPS: 102.14583130087078, Tokens per sec: 83465.98822356318, Loss: 2.277639389038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5835 , TFLOPS: 100.70252099416706, Tokens per sec: 82286.62221784296, Loss: 2.2946903705596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5836 , TFLOPS: 101.52863206084047, Tokens per sec: 82961.65883641255, Loss: 2.2611796855926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5837 , TFLOPS: 101.52823642596236, Tokens per sec: 82961.3355529691, Loss: 2.2922255992889404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5838 , TFLOPS: 101.3790107416907, Tokens per sec: 82839.39940493986, Loss: 2.2821688652038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5839 , TFLOPS: 100.92384452380999, Tokens per sec: 82467.4713712887, Loss: 2.2708992958068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5840 , TFLOPS: 101.66648310428832, Tokens per sec: 83074.30047261534, Loss: 2.285471200942993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5841 , TFLOPS: 100.19232986385387, Tokens per sec: 81869.73191177598, Loss: 2.283323287963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5842 , TFLOPS: 101.83105556402067, Tokens per sec: 83208.77686593465, Loss: 2.292630195617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5843 , TFLOPS: 100.30662582087513, Tokens per sec: 81963.12608049774, Loss: 2.2790098190307617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5844 , TFLOPS: 100.69694053007328, Tokens per sec: 82282.06227697802, Loss: 2.278900146484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5845 , TFLOPS: 102.24189835863366, Tokens per sec: 83544.48708944731, Loss: 2.2936158180236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5846 , TFLOPS: 100.32308414346367, Tokens per sec: 81976.57459955975, Loss: 2.265857219696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5847 , TFLOPS: 102.79916215362154, Tokens per sec: 83999.84168157818, Loss: 2.2903833389282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5848 , TFLOPS: 102.69451396197884, Tokens per sec: 83914.33095030283, Loss: 2.2880260944366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5849 , TFLOPS: 103.39103166326215, Tokens per sec: 84483.47349398215, Loss: 2.287431001663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5850 , TFLOPS: 102.57386160711563, Tokens per sec: 83815.74280527618, Loss: 2.285893440246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5851 , TFLOPS: 102.58384367524285, Tokens per sec: 83823.89941010422, Loss: 2.2809035778045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5852 , TFLOPS: 102.84092261134559, Tokens per sec: 84033.96522659433, Loss: 2.274127960205078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5853 , TFLOPS: 102.80406339321165, Tokens per sec: 84003.84661061638, Loss: 2.2768936157226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5854 , TFLOPS: 101.31498197342626, Tokens per sec: 82787.07985014385, Loss: 2.27759051322937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5855 , TFLOPS: 102.02738936036998, Tokens per sec: 83369.2062650132, Loss: 2.2921435832977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5856 , TFLOPS: 103.34779251485712, Tokens per sec: 84448.14167274568, Loss: 2.268061876296997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5857 , TFLOPS: 102.57666053852483, Tokens per sec: 83818.02988418154, Loss: 2.3020308017730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5858 , TFLOPS: 103.39862879011537, Tokens per sec: 84489.68130190135, Loss: 2.3018314838409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5859 , TFLOPS: 103.3381983188512, Tokens per sec: 84440.30200821267, Loss: 2.2945523262023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5860 , TFLOPS: 100.09013568818081, Tokens per sec: 81786.2264201214, Loss: 2.2754483222961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5861 , TFLOPS: 102.25016849977445, Tokens per sec: 83551.24483466582, Loss: 2.310763120651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5862 , TFLOPS: 102.51558846163631, Tokens per sec: 83768.12631802082, Loss: 2.284898281097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5863 , TFLOPS: 103.3280647618522, Tokens per sec: 84432.02161792784, Loss: 2.3118762969970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5864 , TFLOPS: 101.47486474397238, Tokens per sec: 82917.72417770575, Loss: 2.274204969406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5865 , TFLOPS: 101.66350123234999, Tokens per sec: 83071.86390829431, Loss: 2.2837119102478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5866 , TFLOPS: 103.41260134760195, Tokens per sec: 84501.09863830927, Loss: 2.2700586318969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5867 , TFLOPS: 102.75032148735153, Tokens per sec: 83959.93271589835, Loss: 2.2828421592712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5868 , TFLOPS: 101.00379355689508, Tokens per sec: 82532.79978429373, Loss: 2.2690258026123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5869 , TFLOPS: 102.0982273742756, Tokens per sec: 83427.08982970817, Loss: 2.3145692348480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5870 , TFLOPS: 101.97625421498557, Tokens per sec: 83327.42242138385, Loss: 2.2839744091033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5871 , TFLOPS: 101.5304112098346, Tokens per sec: 82963.11262485501, Loss: 2.272624969482422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5872 , TFLOPS: 102.11138816276198, Tokens per sec: 83437.84384876925, Loss: 2.2834136486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5873 , TFLOPS: 100.15342521816824, Tokens per sec: 81837.94192429158, Loss: 2.2695493698120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5874 , TFLOPS: 102.19837887077787, Tokens per sec: 83508.92619562906, Loss: 2.2831919193267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5875 , TFLOPS: 101.52786315472737, Tokens per sec: 82961.03054343413, Loss: 2.292046308517456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5876 , TFLOPS: 101.54339048369467, Tokens per sec: 82973.7183236423, Loss: 2.29070782661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5877 , TFLOPS: 100.84919303005556, Tokens per sec: 82406.47171404156, Loss: 2.277388334274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5878 , TFLOPS: 101.56608506506025, Tokens per sec: 82992.26265028662, Loss: 2.31345796585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5879 , TFLOPS: 100.4425000258213, Tokens per sec: 82074.15239107245, Loss: 2.285773754119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5880 , TFLOPS: 102.02314018366046, Tokens per sec: 83365.73415334031, Loss: 2.2895383834838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5881 , TFLOPS: 100.3915217976309, Tokens per sec: 82032.49676852174, Loss: 2.288700580596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5882 , TFLOPS: 101.33210824225944, Tokens per sec: 82801.07416527683, Loss: 2.2917301654815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5883 , TFLOPS: 101.41040985774488, Tokens per sec: 82865.05643095289, Loss: 2.287853717803955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5884 , TFLOPS: 101.09672819427216, Tokens per sec: 82608.73906884495, Loss: 2.283656120300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5885 , TFLOPS: 102.72585174001652, Tokens per sec: 83939.93785544358, Loss: 2.2736542224884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5886 , TFLOPS: 102.6387451182571, Tokens per sec: 83868.76079248004, Loss: 2.2920658588409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5887 , TFLOPS: 103.4626623571728, Tokens per sec: 84542.00477791466, Loss: 2.292020082473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5888 , TFLOPS: 102.57039455323465, Tokens per sec: 83812.90978630012, Loss: 2.2928647994995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5889 , TFLOPS: 102.71326229037332, Tokens per sec: 83929.65069205887, Loss: 2.275519609451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5890 , TFLOPS: 102.68139206903986, Tokens per sec: 83903.6087137948, Loss: 2.293099880218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5891 , TFLOPS: 102.64402743642667, Tokens per sec: 83873.07711063535, Loss: 2.2908589839935303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5892 , TFLOPS: 100.60462371217423, Tokens per sec: 82206.6278286264, Loss: 2.303622245788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5893 , TFLOPS: 101.48241450463351, Tokens per sec: 82923.89328149009, Loss: 2.2917425632476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5894 , TFLOPS: 103.20603864465939, Tokens per sec: 84332.31093633786, Loss: 2.263958215713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5895 , TFLOPS: 101.97820296536572, Tokens per sec: 83329.01479548478, Loss: 2.283595085144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5896 , TFLOPS: 102.70705941628941, Tokens per sec: 83924.5821639687, Loss: 2.280785083770752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5897 , TFLOPS: 103.30864676564885, Tokens per sec: 84416.15467336637, Loss: 2.2718615531921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5898 , TFLOPS: 100.59802371150829, Tokens per sec: 82201.23479818313, Loss: 2.2771921157836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5899 , TFLOPS: 101.54198783545411, Tokens per sec: 82972.57218365757, Loss: 2.2844631671905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5900 , TFLOPS: 103.40847985403197, Tokens per sec: 84497.73085981654, Loss: 2.2672932147979736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5901 , TFLOPS: 103.37073530938957, Tokens per sec: 84466.88882075822, Loss: 2.2966084480285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5902 , TFLOPS: 100.92309873863887, Tokens per sec: 82466.86197102717, Loss: 2.2742321491241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5903 , TFLOPS: 101.60885537573301, Tokens per sec: 83027.21137214308, Loss: 2.2813222408294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5904 , TFLOPS: 102.5204800249511, Tokens per sec: 83772.12334032536, Loss: 2.2933948040008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5905 , TFLOPS: 102.0780316368227, Tokens per sec: 83410.58737275089, Loss: 2.276122570037842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5906 , TFLOPS: 100.87974519609135, Tokens per sec: 82431.43667539212, Loss: 2.298628568649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5907 , TFLOPS: 102.13089992930271, Tokens per sec: 83453.78741548723, Loss: 2.269230365753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5908 , TFLOPS: 101.46963762792134, Tokens per sec: 82913.45296662251, Loss: 2.3104100227355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5909 , TFLOPS: 100.90001901348396, Tokens per sec: 82448.0029334781, Loss: 2.3181936740875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5910 , TFLOPS: 102.10046865875768, Tokens per sec: 83428.92124095428, Loss: 2.2850465774536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5911 , TFLOPS: 100.12012307953091, Tokens per sec: 81810.7298895377, Loss: 2.2682065963745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5912 , TFLOPS: 100.83815242534547, Tokens per sec: 82397.4501517225, Loss: 2.2936506271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5913 , TFLOPS: 101.59713160914711, Tokens per sec: 83017.63158066916, Loss: 2.291400671005249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5914 , TFLOPS: 100.9520907158058, Tokens per sec: 82490.55206189015, Loss: 2.292006492614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5915 , TFLOPS: 99.7888910312086, Tokens per sec: 81540.07165618207, Loss: 2.29335355758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5916 , TFLOPS: 101.65790282882749, Tokens per sec: 83067.28930866034, Loss: 2.296647548675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5917 , TFLOPS: 100.45461404391382, Tokens per sec: 82084.05106709842, Loss: 2.2863597869873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5918 , TFLOPS: 101.9399092069301, Tokens per sec: 83297.72397969798, Loss: 2.3208329677581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5919 , TFLOPS: 100.10843971177951, Tokens per sec: 81801.18310899139, Loss: 2.2736809253692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5920 , TFLOPS: 100.71840057094165, Tokens per sec: 82299.59782880216, Loss: 2.2788186073303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5921 , TFLOPS: 102.2544490536705, Tokens per sec: 83554.74258544555, Loss: 2.2945399284362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5922 , TFLOPS: 101.19953452495172, Tokens per sec: 82692.7447681136, Loss: 2.275177001953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5923 , TFLOPS: 102.08879012948084, Tokens per sec: 83419.3784140501, Loss: 2.264092206954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5924 , TFLOPS: 102.76307874000983, Tokens per sec: 83970.35699544635, Loss: 2.2770636081695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5925 , TFLOPS: 102.49857147894994, Tokens per sec: 83754.22130341167, Loss: 2.2828402519226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5926 , TFLOPS: 102.68406440399299, Tokens per sec: 83905.79234747702, Loss: 2.3171145915985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5927 , TFLOPS: 102.62541245142347, Tokens per sec: 83857.86632719691, Loss: 2.314913749694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5928 , TFLOPS: 102.66192513961872, Tokens per sec: 83887.70178464145, Loss: 2.2828152179718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5929 , TFLOPS: 102.6066053425729, Tokens per sec: 83842.49855441706, Loss: 2.2923731803894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5930 , TFLOPS: 100.88590122162442, Tokens per sec: 82436.46692232539, Loss: 2.2717065811157227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5931 , TFLOPS: 101.95408390210537, Tokens per sec: 83309.3064880148, Loss: 2.286055088043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5932 , TFLOPS: 103.43255367065896, Tokens per sec: 84517.40219509765, Loss: 2.284205436706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5933 , TFLOPS: 102.84255704083938, Tokens per sec: 84035.30076100766, Loss: 2.2758328914642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5934 , TFLOPS: 102.77790806479187, Tokens per sec: 83982.47441846653, Loss: 2.2875359058380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5935 , TFLOPS: 103.33047019604128, Tokens per sec: 84433.9871601251, Loss: 2.2782139778137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5936 , TFLOPS: 100.45871136682341, Tokens per sec: 82087.39909512231, Loss: 2.2874460220336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5937 , TFLOPS: 101.48855198925696, Tokens per sec: 82928.90837816882, Loss: 2.320246458053589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5938 , TFLOPS: 103.39982455343133, Tokens per sec: 84490.65839088851, Loss: 2.3049731254577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5939 , TFLOPS: 102.5427902854802, Tokens per sec: 83790.3536285209, Loss: 2.2658724784851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5940 , TFLOPS: 100.90041490982772, Tokens per sec: 82448.32643057188, Loss: 2.2944819927215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5941 , TFLOPS: 101.7445231767984, Tokens per sec: 83138.0690247935, Loss: 2.3052330017089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5942 , TFLOPS: 103.4185292729092, Tokens per sec: 84505.94249867619, Loss: 2.2712321281433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5943 , TFLOPS: 102.20159510379712, Tokens per sec: 83511.5542624223, Loss: 2.2816810607910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5944 , TFLOPS: 101.00260562201944, Tokens per sec: 82531.82909212669, Loss: 2.3040854930877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5945 , TFLOPS: 101.3719683616009, Tokens per sec: 82833.6448948815, Loss: 2.2758677005767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5946 , TFLOPS: 102.12491945442005, Tokens per sec: 83448.90061550948, Loss: 2.293351173400879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5947 , TFLOPS: 100.94162227710443, Tokens per sec: 82481.99803114569, Loss: 2.3086400032043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5948 , TFLOPS: 101.96775348966148, Tokens per sec: 83320.47625989339, Loss: 2.277513027191162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5949 , TFLOPS: 100.24091189991535, Tokens per sec: 81909.42954405467, Loss: 2.279616594314575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5950 , TFLOPS: 100.93793223804454, Tokens per sec: 82478.9828052398, Loss: 2.271311044692993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5951 , TFLOPS: 101.6973204862664, Tokens per sec: 83099.49849125446, Loss: 2.284440517425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5952 , TFLOPS: 101.1080290198264, Tokens per sec: 82617.97326431445, Loss: 2.2794265747070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5953 , TFLOPS: 101.16215678635164, Tokens per sec: 82662.2024557149, Loss: 2.286503553390503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5954 , TFLOPS: 100.38490740701002, Tokens per sec: 82027.09197967585, Loss: 2.3095598220825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5955 , TFLOPS: 100.08860184238029, Tokens per sec: 81784.9730752331, Loss: 2.2968225479125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5956 , TFLOPS: 102.67030724152467, Tokens per sec: 83894.5510158833, Loss: 2.2748360633850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5957 , TFLOPS: 99.01732562862676, Tokens per sec: 80909.60570387202, Loss: 2.2811758518218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5958 , TFLOPS: 101.451122307753, Tokens per sec: 82898.32362188621, Loss: 2.2806787490844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5959 , TFLOPS: 102.24761354953351, Tokens per sec: 83549.15711905368, Loss: 2.2641959190368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5960 , TFLOPS: 100.61683579835423, Tokens per sec: 82216.60663861113, Loss: 2.2858591079711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5961 , TFLOPS: 102.7009790421729, Tokens per sec: 83919.61373375535, Loss: 2.292045831680298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5962 , TFLOPS: 101.59727926861986, Tokens per sec: 83017.75223702542, Loss: 2.2883763313293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5963 , TFLOPS: 103.3869251362856, Tokens per sec: 84480.1179450782, Loss: 2.29548978805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5964 , TFLOPS: 102.72697443188021, Tokens per sec: 83940.85523586607, Loss: 2.28088641166687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5965 , TFLOPS: 102.62877297441491, Tokens per sec: 83860.61229704077, Loss: 2.298840045928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5966 , TFLOPS: 102.66632202077602, Tokens per sec: 83891.29458942087, Loss: 2.285531520843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5967 , TFLOPS: 102.64690354366586, Tokens per sec: 83875.42725189809, Loss: 2.292757034301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5968 , TFLOPS: 100.12984065287264, Tokens per sec: 81818.67036886781, Loss: 2.263256549835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5969 , TFLOPS: 102.60675522289401, Tokens per sec: 83842.62102548569, Loss: 2.3211441040039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5970 , TFLOPS: 102.873714074478, Tokens per sec: 84060.75997524717, Loss: 2.2752301692962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5971 , TFLOPS: 102.07938241837894, Tokens per sec: 83411.6911311327, Loss: 2.3136119842529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5972 , TFLOPS: 100.32420111602106, Tokens per sec: 81977.48730658981, Loss: 2.2835757732391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5973 , TFLOPS: 103.4396140964033, Tokens per sec: 84523.17145072487, Loss: 2.284930944442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5974 , TFLOPS: 103.38527971597956, Tokens per sec: 84478.77342978925, Loss: 2.289215564727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5975 , TFLOPS: 100.90899360234539, Tokens per sec: 82455.33630106326, Loss: 2.2875332832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5976 , TFLOPS: 101.68069072409723, Tokens per sec: 83085.90988449786, Loss: 2.311311721801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5977 , TFLOPS: 103.35913579572345, Tokens per sec: 84457.41055954358, Loss: 2.277604818344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5978 , TFLOPS: 101.39951890924179, Tokens per sec: 82856.15715657306, Loss: 2.2578420639038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5979 , TFLOPS: 101.03113633984609, Tokens per sec: 82555.14227610886, Loss: 2.287261962890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5980 , TFLOPS: 102.63176157414577, Tokens per sec: 83863.05435881425, Loss: 2.268570899963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5981 , TFLOPS: 100.04678322809448, Tokens per sec: 81750.80200899325, Loss: 2.278723955154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5982 , TFLOPS: 102.7365828990676, Tokens per sec: 83948.70656174883, Loss: 2.2850112915039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5983 , TFLOPS: 99.94784408260371, Tokens per sec: 81669.95628629257, Loss: 2.3015031814575195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5984 , TFLOPS: 102.7274972952095, Tokens per sec: 83941.28248095214, Loss: 2.2912228107452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5985 , TFLOPS: 100.25154896420685, Tokens per sec: 81918.12136310965, Loss: 2.2896554470062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5986 , TFLOPS: 102.1169237639931, Tokens per sec: 83442.36713103425, Loss: 2.2992019653320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5987 , TFLOPS: 97.78989782343044, Tokens per sec: 79906.64284744347, Loss: 2.28953218460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5988 , TFLOPS: 101.13954587742816, Tokens per sec: 82643.72649997694, Loss: 2.2663159370422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5989 , TFLOPS: 101.73853995457075, Tokens per sec: 83133.17997988945, Loss: 2.318737030029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5990 , TFLOPS: 99.7781592279156, Tokens per sec: 81531.30242345015, Loss: 2.301344156265259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5991 , TFLOPS: 101.47459687341721, Tokens per sec: 82917.50529377953, Loss: 2.2835731506347656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5992 , TFLOPS: 100.57958902489317, Tokens per sec: 82186.17134119886, Loss: 2.2968103885650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5993 , TFLOPS: 102.83758139043846, Tokens per sec: 84031.23502898043, Loss: 2.2552220821380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5994 , TFLOPS: 102.72323717857539, Tokens per sec: 83937.80142998524, Loss: 2.2892518043518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5995 , TFLOPS: 102.0191465947539, Tokens per sec: 83362.47088904065, Loss: 2.273815870285034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5996 , TFLOPS: 102.82079785600546, Tokens per sec: 84017.52077094861, Loss: 2.268822193145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5997 , TFLOPS: 102.90357298368345, Tokens per sec: 84085.15845859564, Loss: 2.2753279209136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5998 , TFLOPS: 102.62809803668668, Tokens per sec: 83860.06078804858, Loss: 2.2991695404052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5999 , TFLOPS: 101.54236652139, Tokens per sec: 82972.88161768403, Loss: 2.278054714202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6000 , TFLOPS: 101.23570041627761, Tokens per sec: 82722.29684890887, Loss: 2.2712771892547607 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/model.pt +[rank0]:[2024-08-28 00:44:02,780] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007731925987172872, 'preprocessing_with_comm': 0.0015285910048987716, 'state_converting': 3.291814940981567, : 3.3027069849777035}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000, took 15.75s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004000 + eval ppl=7.707389831542969, eval loss=2.042179584503174 +------------------------------------------------------------------ +iteration: 6001 , TFLOPS: 101.64285936249215, Tokens per sec: 83054.99690506446, Loss: 2.264435291290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6002 , TFLOPS: 100.03786174262318, Tokens per sec: 81743.51203355519, Loss: 2.2898433208465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6003 , TFLOPS: 101.55416496812656, Tokens per sec: 82982.52243223152, Loss: 2.286353588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6004 , TFLOPS: 101.78061335919314, Tokens per sec: 83167.55924187218, Loss: 2.2847580909729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6005 , TFLOPS: 100.94936659999277, Tokens per sec: 82488.32611673434, Loss: 2.287446975708008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6006 , TFLOPS: 102.22808330535021, Tokens per sec: 83533.1984537783, Loss: 2.296102285385132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6007 , TFLOPS: 102.3343448763408, Tokens per sec: 83620.02751885088, Loss: 2.2724366188049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6008 , TFLOPS: 102.59338184446825, Tokens per sec: 83831.69329371261, Loss: 2.292163372039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6009 , TFLOPS: 102.73287879173787, Tokens per sec: 83945.67984029751, Loss: 2.2729084491729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6010 , TFLOPS: 103.2026440058294, Tokens per sec: 84329.53708956411, Loss: 2.2769219875335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6011 , TFLOPS: 102.47552112961145, Tokens per sec: 83735.38626959841, Loss: 2.28094744682312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6012 , TFLOPS: 103.30882597029156, Tokens per sec: 84416.30110609307, Loss: 2.2800040245056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6013 , TFLOPS: 101.86288152019128, Tokens per sec: 83234.78267399465, Loss: 2.294490098953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6014 , TFLOPS: 103.30950368258435, Tokens per sec: 84416.85488225339, Loss: 2.2728514671325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6015 , TFLOPS: 102.68417516826341, Tokens per sec: 83905.88285581372, Loss: 2.2748236656188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6016 , TFLOPS: 102.63693937352159, Tokens per sec: 83867.28527197264, Loss: 2.283263683319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6017 , TFLOPS: 102.66163117461062, Tokens per sec: 83887.461578266, Loss: 2.279825448989868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6018 , TFLOPS: 102.57794622970985, Tokens per sec: 83819.08045554545, Loss: 2.27372145652771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6019 , TFLOPS: 101.97958347217586, Tokens per sec: 83330.14284313674, Loss: 2.299454689025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6020 , TFLOPS: 103.41149008077589, Tokens per sec: 84500.19059357913, Loss: 2.3118057250976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6021 , TFLOPS: 102.62970058349092, Tokens per sec: 83861.37027029555, Loss: 2.269749402999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6022 , TFLOPS: 102.0113855221934, Tokens per sec: 83356.12911686367, Loss: 2.277733325958252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6023 , TFLOPS: 101.48676927153407, Tokens per sec: 82927.45167362622, Loss: 2.287465810775757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6024 , TFLOPS: 103.34524124565088, Tokens per sec: 84446.05696500177, Loss: 2.2788424491882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6025 , TFLOPS: 102.78350948480558, Tokens per sec: 83987.05148295312, Loss: 2.3043158054351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6026 , TFLOPS: 101.89377270132135, Tokens per sec: 83260.02465330577, Loss: 2.2895095348358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6027 , TFLOPS: 101.79124108208198, Tokens per sec: 83176.2434278266, Loss: 2.274153709411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6028 , TFLOPS: 103.37129280452672, Tokens per sec: 84467.344364386, Loss: 2.27164888381958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6029 , TFLOPS: 101.00960120319428, Tokens per sec: 82537.54536158693, Loss: 2.2746407985687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6030 , TFLOPS: 102.19406247470404, Tokens per sec: 83505.39915728322, Loss: 2.2880868911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6031 , TFLOPS: 103.29161878827504, Tokens per sec: 84402.24067471511, Loss: 2.2950339317321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6032 , TFLOPS: 100.1817180698091, Tokens per sec: 81861.0607416901, Loss: 2.2893991470336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6033 , TFLOPS: 102.79801647359469, Tokens per sec: 83998.90551693582, Loss: 2.2792108058929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6034 , TFLOPS: 101.5257865265594, Tokens per sec: 82959.3336771009, Loss: 2.2944564819335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6035 , TFLOPS: 102.02655052712774, Tokens per sec: 83368.52083277747, Loss: 2.277151584625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6036 , TFLOPS: 100.88840558622447, Tokens per sec: 82438.51330310818, Loss: 2.2693660259246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6037 , TFLOPS: 102.05533297735755, Tokens per sec: 83392.03971378643, Loss: 2.294431686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6038 , TFLOPS: 100.7837801325116, Tokens per sec: 82353.0211515806, Loss: 2.2864933013916016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6039 , TFLOPS: 102.0273426037223, Tokens per sec: 83369.16805895277, Loss: 2.289518117904663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6040 , TFLOPS: 100.84584169147045, Tokens per sec: 82403.73324901254, Loss: 2.279911518096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6041 , TFLOPS: 100.96210700569914, Tokens per sec: 82498.73663020425, Loss: 2.327134609222412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6042 , TFLOPS: 101.47005741747593, Tokens per sec: 82913.7959874739, Loss: 2.2919232845306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6043 , TFLOPS: 101.1492564811872, Tokens per sec: 82651.66128428162, Loss: 2.3008737564086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6044 , TFLOPS: 101.70663576733394, Tokens per sec: 83107.1102472114, Loss: 2.283090353012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6045 , TFLOPS: 102.32301964415187, Tokens per sec: 83610.77338009191, Loss: 2.2851526737213135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6046 , TFLOPS: 102.76927193830711, Tokens per sec: 83975.41761720217, Loss: 2.285926580429077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6047 , TFLOPS: 102.70466822289063, Tokens per sec: 83922.62825828778, Loss: 2.27801775932312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6048 , TFLOPS: 103.34734787178186, Tokens per sec: 84447.77834344277, Loss: 2.269157886505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6049 , TFLOPS: 103.3771103931938, Tokens per sec: 84472.09806584392, Loss: 2.3025147914886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6050 , TFLOPS: 103.3191250882049, Tokens per sec: 84424.71678046294, Loss: 2.2748193740844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6051 , TFLOPS: 102.00645191730716, Tokens per sec: 83352.09774130884, Loss: 2.2876148223876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6052 , TFLOPS: 103.4263886814444, Tokens per sec: 84512.36462370948, Loss: 2.281083583831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6053 , TFLOPS: 101.86534365701515, Tokens per sec: 83236.79454937446, Loss: 2.2730703353881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6054 , TFLOPS: 102.75188531604432, Tokens per sec: 83961.21056057977, Loss: 2.2850770950317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6055 , TFLOPS: 102.68282407400652, Tokens per sec: 83904.77884191615, Loss: 2.3021628856658936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6056 , TFLOPS: 103.35608489148812, Tokens per sec: 84454.91758715565, Loss: 2.283179998397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6057 , TFLOPS: 102.07115876861658, Tokens per sec: 83404.97137521631, Loss: 2.269425630569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6058 , TFLOPS: 103.34677706488623, Tokens per sec: 84447.31192243446, Loss: 2.27475643157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6059 , TFLOPS: 101.75727341166603, Tokens per sec: 83148.4875699241, Loss: 2.2864491939544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6060 , TFLOPS: 102.7419813428572, Tokens per sec: 83953.11776913762, Loss: 2.2609140872955322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6061 , TFLOPS: 101.41090549930453, Tokens per sec: 82865.46143243022, Loss: 2.281357526779175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6062 , TFLOPS: 103.3725037685635, Tokens per sec: 84468.33387427343, Loss: 2.3046889305114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6063 , TFLOPS: 102.68351899207526, Tokens per sec: 83905.3466773589, Loss: 2.278158187866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6064 , TFLOPS: 102.04872259408432, Tokens per sec: 83386.63819944748, Loss: 2.2973828315734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6065 , TFLOPS: 101.57678721539585, Tokens per sec: 83001.0076528237, Loss: 2.278803825378418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6066 , TFLOPS: 103.37024462280765, Tokens per sec: 84466.48786812075, Loss: 2.283770799636841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6067 , TFLOPS: 101.41598079718621, Tokens per sec: 82869.60858897913, Loss: 2.285773754119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6068 , TFLOPS: 101.38992913077907, Tokens per sec: 82848.3211017283, Loss: 2.265658140182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6069 , TFLOPS: 103.3574211933702, Tokens per sec: 84456.00951382297, Loss: 2.2736141681671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6070 , TFLOPS: 100.71461396640454, Tokens per sec: 82296.50369675901, Loss: 2.316555976867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6071 , TFLOPS: 102.02965053908973, Tokens per sec: 83371.05393235302, Loss: 2.292968511581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6072 , TFLOPS: 101.55969573582092, Tokens per sec: 82987.0417648892, Loss: 2.2961642742156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6073 , TFLOPS: 102.73240649294763, Tokens per sec: 83945.29391279795, Loss: 2.277589797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6074 , TFLOPS: 100.0975791077096, Tokens per sec: 81792.30862982906, Loss: 2.270282745361328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6075 , TFLOPS: 102.89135641863288, Tokens per sec: 84075.17598881047, Loss: 2.257852077484131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6076 , TFLOPS: 100.76899487510791, Tokens per sec: 82340.93973714966, Loss: 2.285956382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6077 , TFLOPS: 102.62541725934108, Tokens per sec: 83857.87025587015, Loss: 2.276852607727051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6078 , TFLOPS: 100.29310950709834, Tokens per sec: 81952.08155256984, Loss: 2.269479751586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6079 , TFLOPS: 101.6879552164545, Tokens per sec: 83091.84588820767, Loss: 2.2993075847625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6080 , TFLOPS: 101.47802663262124, Tokens per sec: 82920.307838315, Loss: 2.2898447513580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6081 , TFLOPS: 101.05268015845112, Tokens per sec: 82572.74628487827, Loss: 2.2815494537353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6082 , TFLOPS: 101.51764387573645, Tokens per sec: 82952.68010750301, Loss: 2.279660701751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6083 , TFLOPS: 102.33435869695029, Tokens per sec: 83620.03881202669, Loss: 2.240243673324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6084 , TFLOPS: 102.21575206644593, Tokens per sec: 83523.12228103528, Loss: 2.267940044403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6085 , TFLOPS: 102.17877552315807, Tokens per sec: 83492.90779565387, Loss: 2.284790515899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6086 , TFLOPS: 103.36135171851177, Tokens per sec: 84459.22124709687, Loss: 2.280926465988159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6087 , TFLOPS: 103.38414192990383, Tokens per sec: 84477.84371549755, Loss: 2.2623767852783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6088 , TFLOPS: 103.32600975542432, Tokens per sec: 84430.34241927504, Loss: 2.296588897705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6089 , TFLOPS: 101.25054019488955, Tokens per sec: 82734.42281402256, Loss: 2.278818368911743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6090 , TFLOPS: 103.36756186080989, Tokens per sec: 84464.29571423194, Loss: 2.275843381881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6091 , TFLOPS: 102.61023788952325, Tokens per sec: 83845.46680204026, Loss: 2.278917074203491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6092 , TFLOPS: 102.66571039016944, Tokens per sec: 83890.79481030746, Loss: 2.2892005443573 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6093 , TFLOPS: 101.68264409991626, Tokens per sec: 83087.50603816395, Loss: 2.2637460231781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6094 , TFLOPS: 102.57877033265225, Tokens per sec: 83819.753851274, Loss: 2.281373977661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6095 , TFLOPS: 101.9511359400781, Tokens per sec: 83306.89763235503, Loss: 2.2915804386138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6096 , TFLOPS: 102.82478375253761, Tokens per sec: 84020.77774961114, Loss: 2.2805657386779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6097 , TFLOPS: 102.62949848480237, Tokens per sec: 83861.20513025465, Loss: 2.280998945236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6098 , TFLOPS: 102.83673515598832, Tokens per sec: 84030.5435490272, Loss: 2.278693675994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6099 , TFLOPS: 100.18459543434662, Tokens per sec: 81863.41191032367, Loss: 2.2837018966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6100 , TFLOPS: 103.36609238729554, Tokens per sec: 84463.09496959575, Loss: 2.3036937713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6101 , TFLOPS: 102.67546409920362, Tokens per sec: 83898.7648170421, Loss: 2.2937564849853516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6102 , TFLOPS: 102.10868200613402, Tokens per sec: 83435.63257852587, Loss: 2.2880935668945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6103 , TFLOPS: 101.03009588110758, Tokens per sec: 82554.29209048983, Loss: 2.274393320083618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6104 , TFLOPS: 102.89412275613532, Tokens per sec: 84077.43643439557, Loss: 2.2678608894348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6105 , TFLOPS: 101.60830950767291, Tokens per sec: 83026.76532929928, Loss: 2.2668774127960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6106 , TFLOPS: 100.88002769490845, Tokens per sec: 82431.66751244773, Loss: 2.29864239692688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6107 , TFLOPS: 103.2766222423215, Tokens per sec: 84389.98661097103, Loss: 2.2824530601501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6108 , TFLOPS: 100.21339093913022, Tokens per sec: 81886.9414585446, Loss: 2.2843573093414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6109 , TFLOPS: 101.40535442016132, Tokens per sec: 82860.92550276482, Loss: 2.3063247203826904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6110 , TFLOPS: 101.64973811695555, Tokens per sec: 83060.6177124114, Loss: 2.2990121841430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6111 , TFLOPS: 101.35046149565889, Tokens per sec: 82816.0710810843, Loss: 2.2964563369750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6112 , TFLOPS: 100.21946460323406, Tokens per sec: 81891.90440583386, Loss: 2.282142162322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6113 , TFLOPS: 102.1801457036038, Tokens per sec: 83494.02740536765, Loss: 2.307293176651001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6114 , TFLOPS: 100.85726362950933, Tokens per sec: 82413.06640861095, Loss: 2.2742605209350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6115 , TFLOPS: 101.86116843251288, Tokens per sec: 83233.38286595381, Loss: 2.274138927459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6116 , TFLOPS: 100.91794547525718, Tokens per sec: 82462.65110686119, Loss: 2.260509490966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6117 , TFLOPS: 101.11513120124681, Tokens per sec: 82623.77663957948, Loss: 2.292999267578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6118 , TFLOPS: 102.20996481931927, Tokens per sec: 83518.39337243109, Loss: 2.296154260635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6119 , TFLOPS: 100.13448015906667, Tokens per sec: 81822.46142881048, Loss: 2.2705702781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6120 , TFLOPS: 100.98122572503839, Tokens per sec: 82514.35902793596, Loss: 2.255336284637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6121 , TFLOPS: 102.39289338665453, Tokens per sec: 83667.86901380097, Loss: 2.2845935821533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6122 , TFLOPS: 102.63816984298343, Tokens per sec: 83868.29072024502, Loss: 2.282066583633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6123 , TFLOPS: 102.84817427206642, Tokens per sec: 84039.89074524365, Loss: 2.294971466064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6124 , TFLOPS: 103.43761670945987, Tokens per sec: 84521.53933443608, Loss: 2.275878667831421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6125 , TFLOPS: 102.86532806765258, Tokens per sec: 84053.9075531951, Loss: 2.2952215671539307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6126 , TFLOPS: 103.33016255504427, Tokens per sec: 84433.73577874707, Loss: 2.282492160797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6127 , TFLOPS: 102.02488119340246, Tokens per sec: 83367.15677721807, Loss: 2.2895708084106445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6128 , TFLOPS: 102.79553796251075, Tokens per sec: 83996.88026172658, Loss: 2.3070130348205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6129 , TFLOPS: 102.63955964136063, Tokens per sec: 83869.42636027563, Loss: 2.2819864749908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6130 , TFLOPS: 101.81506075251723, Tokens per sec: 83195.7070937121, Loss: 2.273064374923706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6131 , TFLOPS: 101.43907750526711, Tokens per sec: 82888.4815037142, Loss: 2.311445474624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6132 , TFLOPS: 103.34489792996904, Tokens per sec: 84445.7764329204, Loss: 2.280460834503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6133 , TFLOPS: 102.5956766894477, Tokens per sec: 83833.5684706199, Loss: 2.284008502960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6134 , TFLOPS: 102.20917222752973, Tokens per sec: 83517.74572527673, Loss: 2.2895641326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6135 , TFLOPS: 102.60830420606398, Tokens per sec: 83843.88673950825, Loss: 2.2745137214660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6136 , TFLOPS: 101.2646935238158, Tokens per sec: 82745.98786342729, Loss: 2.2686290740966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6137 , TFLOPS: 101.37903424654218, Tokens per sec: 82839.41861135907, Loss: 2.280846357345581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6138 , TFLOPS: 102.76171173737912, Tokens per sec: 83969.23998240702, Loss: 2.276160717010498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6139 , TFLOPS: 102.71431713971788, Tokens per sec: 83930.51263661253, Loss: 2.27406907081604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6140 , TFLOPS: 100.95796683528279, Tokens per sec: 82495.35359037957, Loss: 2.2629904747009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6141 , TFLOPS: 101.56377813812485, Tokens per sec: 82990.37760090212, Loss: 2.2765579223632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6142 , TFLOPS: 102.87453364295705, Tokens per sec: 84061.42966574898, Loss: 2.294478416442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6143 , TFLOPS: 101.16768965282914, Tokens per sec: 82666.7235033423, Loss: 2.267479658126831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6144 , TFLOPS: 100.99035144963568, Tokens per sec: 82521.81589242147, Loss: 2.29624080657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6145 , TFLOPS: 103.33479004621991, Tokens per sec: 84437.5170209089, Loss: 2.2965586185455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6146 , TFLOPS: 100.73848227654213, Tokens per sec: 82316.0071073974, Loss: 2.283982038497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6147 , TFLOPS: 102.09730055075553, Tokens per sec: 83426.33249835137, Loss: 2.268582344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6148 , TFLOPS: 101.61763770580717, Tokens per sec: 83034.38764012397, Loss: 2.2877469062805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6149 , TFLOPS: 102.17545020493098, Tokens per sec: 83490.19059252921, Loss: 2.295806646347046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6150 , TFLOPS: 100.75153535483479, Tokens per sec: 82326.67311368645, Loss: 2.2699224948883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6151 , TFLOPS: 102.08550785650084, Tokens per sec: 83416.69638430492, Loss: 2.279404640197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6152 , TFLOPS: 101.50136586034799, Tokens per sec: 82939.3789220957, Loss: 2.28578782081604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6153 , TFLOPS: 101.34385731968283, Tokens per sec: 82810.67463888763, Loss: 2.2925593852996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6154 , TFLOPS: 101.38301697098157, Tokens per sec: 82842.67299802296, Loss: 2.2881360054016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6155 , TFLOPS: 101.60928780235622, Tokens per sec: 83027.56471906856, Loss: 2.293210029602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6156 , TFLOPS: 102.07189580522997, Tokens per sec: 83405.57362680612, Loss: 2.2767980098724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6157 , TFLOPS: 100.99194670029695, Tokens per sec: 82523.11941280219, Loss: 2.2933053970336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6158 , TFLOPS: 101.61650655695074, Tokens per sec: 83033.46334927501, Loss: 2.273674249649048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6159 , TFLOPS: 100.95149313179799, Tokens per sec: 82490.0637606142, Loss: 2.291527509689331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6160 , TFLOPS: 102.18119971703267, Tokens per sec: 83494.8886668731, Loss: 2.260390043258667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6161 , TFLOPS: 102.79303995960237, Tokens per sec: 83994.83907924582, Loss: 2.2565832138061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6162 , TFLOPS: 102.727293378272, Tokens per sec: 83941.11585517321, Loss: 2.278306722640991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6163 , TFLOPS: 102.36821007443582, Tokens per sec: 83647.6996439821, Loss: 2.285517692565918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6164 , TFLOPS: 103.51220299954784, Tokens per sec: 84582.48571208886, Loss: 2.2823212146759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6165 , TFLOPS: 102.70707758180662, Tokens per sec: 83924.59700748044, Loss: 2.286400318145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6166 , TFLOPS: 102.05883674775252, Tokens per sec: 83394.90273477124, Loss: 2.2890872955322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6167 , TFLOPS: 102.70767297202539, Tokens per sec: 83925.08351615492, Loss: 2.2832579612731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6168 , TFLOPS: 102.87491658567505, Tokens per sec: 84061.74257810168, Loss: 2.2614376544952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6169 , TFLOPS: 101.64690405913828, Tokens per sec: 83058.30193081377, Loss: 2.2724101543426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6170 , TFLOPS: 103.36627539123302, Tokens per sec: 84463.24450682399, Loss: 2.283623695373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6171 , TFLOPS: 101.90519531433253, Tokens per sec: 83269.35836444142, Loss: 2.282717227935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6172 , TFLOPS: 102.14429513149652, Tokens per sec: 83464.73298002298, Loss: 2.2785277366638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6173 , TFLOPS: 102.03627061626403, Tokens per sec: 83376.4633678281, Loss: 2.3049073219299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6174 , TFLOPS: 101.5695608795199, Tokens per sec: 82995.10282775703, Loss: 2.280271530151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6175 , TFLOPS: 102.06445225390631, Tokens per sec: 83399.4913094055, Loss: 2.301393508911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6176 , TFLOPS: 102.21366589774958, Tokens per sec: 83521.41761889073, Loss: 2.254711627960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6177 , TFLOPS: 101.83361306694698, Tokens per sec: 83210.86666741171, Loss: 2.2869396209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6178 , TFLOPS: 101.59463290984206, Tokens per sec: 83015.58982914475, Loss: 2.2669472694396973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6179 , TFLOPS: 101.02785573966625, Tokens per sec: 82552.46161325175, Loss: 2.2812001705169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6180 , TFLOPS: 103.22771021851612, Tokens per sec: 84350.01933721214, Loss: 2.3099284172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6181 , TFLOPS: 100.55504834901626, Tokens per sec: 82166.11852320663, Loss: 2.2926342487335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6182 , TFLOPS: 100.7959260524821, Tokens per sec: 82362.9458954523, Loss: 2.273475408554077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6183 , TFLOPS: 103.42563268690289, Tokens per sec: 84511.74688110857, Loss: 2.2891530990600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6184 , TFLOPS: 101.49047955274101, Tokens per sec: 82930.48343991155, Loss: 2.275512933731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6185 , TFLOPS: 102.75158667362662, Tokens per sec: 83960.96653217252, Loss: 2.318281650543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6186 , TFLOPS: 101.4285962444348, Tokens per sec: 82879.91699567647, Loss: 2.2899227142333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6187 , TFLOPS: 102.71572969324009, Tokens per sec: 83931.6668704579, Loss: 2.2903037071228027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6188 , TFLOPS: 101.4631224347984, Tokens per sec: 82908.12923460576, Loss: 2.2903194427490234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6189 , TFLOPS: 102.21985228758807, Tokens per sec: 83526.47267727961, Loss: 2.266613721847534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6190 , TFLOPS: 100.90620318415372, Tokens per sec: 82453.0561785271, Loss: 2.329747200012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6191 , TFLOPS: 102.05911430283113, Tokens per sec: 83395.12953217087, Loss: 2.28732967376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6192 , TFLOPS: 100.36583869419607, Tokens per sec: 82011.51044356324, Loss: 2.273033380508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6193 , TFLOPS: 102.25488180882395, Tokens per sec: 83555.09620082154, Loss: 2.282864809036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6194 , TFLOPS: 101.72376670305346, Tokens per sec: 83121.10837577734, Loss: 2.298128366470337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6195 , TFLOPS: 100.6651160733447, Tokens per sec: 82256.0576941507, Loss: 2.2873165607452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6196 , TFLOPS: 100.8761469117417, Tokens per sec: 82428.49642462195, Loss: 2.3034629821777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6197 , TFLOPS: 102.36473184192748, Tokens per sec: 83644.85749066186, Loss: 2.2848758697509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6198 , TFLOPS: 101.40418418120795, Tokens per sec: 82859.96927039139, Loss: 2.2945215702056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6199 , TFLOPS: 102.84333240618179, Tokens per sec: 84035.93433198862, Loss: 2.2897822856903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6200 , TFLOPS: 102.54782753839501, Tokens per sec: 83794.46969754802, Loss: 2.285817861557007 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/model.pt +[rank0]:[2024-08-28 01:36:50,909] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007737237989204004, 'preprocessing_with_comm': 0.0015400740085169673, 'state_converting': 3.3369395140034612, : 3.3478382649773266}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200, took 15.66s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004200 + eval ppl=7.988202095031738, eval loss=2.07796573638916 +------------------------------------------------------------------ +iteration: 6201 , TFLOPS: 101.30277958443153, Tokens per sec: 82777.10896397884, Loss: 2.276193141937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6202 , TFLOPS: 100.73630869348604, Tokens per sec: 82314.23101672926, Loss: 2.282332420349121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6203 , TFLOPS: 100.9146655492609, Tokens per sec: 82459.97099489682, Loss: 2.2851321697235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6204 , TFLOPS: 101.541538566228, Tokens per sec: 82972.2050742077, Loss: 2.2677009105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6205 , TFLOPS: 101.05532423155316, Tokens per sec: 82574.90682507443, Loss: 2.302957773208618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6206 , TFLOPS: 101.95202720648015, Tokens per sec: 83307.62590907539, Loss: 2.29561448097229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6207 , TFLOPS: 101.7287031194757, Tokens per sec: 83125.14204871062, Loss: 2.2863564491271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6208 , TFLOPS: 99.7898805544768, Tokens per sec: 81540.8802211171, Loss: 2.2819995880126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6209 , TFLOPS: 102.6343244287538, Tokens per sec: 83865.14853328823, Loss: 2.2836191654205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6210 , TFLOPS: 103.39727071465856, Tokens per sec: 84488.57158348562, Loss: 2.2578492164611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6211 , TFLOPS: 102.46497371012964, Tokens per sec: 83726.7677016251, Loss: 2.2928614616394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6212 , TFLOPS: 102.59605830607997, Tokens per sec: 83833.8802993938, Loss: 2.2803304195404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6213 , TFLOPS: 102.56692860515969, Tokens per sec: 83810.07767090663, Loss: 2.289599895477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6214 , TFLOPS: 102.44482541905101, Tokens per sec: 83710.30401432159, Loss: 2.2868988513946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6215 , TFLOPS: 102.0227063474085, Tokens per sec: 83365.37965457089, Loss: 2.2776153087615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6216 , TFLOPS: 102.01107836179561, Tokens per sec: 83355.87812819563, Loss: 2.2787394523620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6217 , TFLOPS: 101.97779488114514, Tokens per sec: 83328.68133936312, Loss: 2.2879507541656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6218 , TFLOPS: 102.02522595998293, Tokens per sec: 83367.43849486606, Loss: 2.2893035411834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6219 , TFLOPS: 102.6325468715955, Tokens per sec: 83863.69604557577, Loss: 2.287548542022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6220 , TFLOPS: 102.75587490059998, Tokens per sec: 83964.47055282125, Loss: 2.2485315799713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6221 , TFLOPS: 103.16273829961075, Tokens per sec: 84296.9291097486, Loss: 2.310901403427124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6222 , TFLOPS: 101.35067110820334, Tokens per sec: 82816.24236089028, Loss: 2.2878007888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6223 , TFLOPS: 101.66420562133902, Tokens per sec: 83072.43948266983, Loss: 2.2800040245056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6224 , TFLOPS: 103.49608950755444, Tokens per sec: 84569.31896298288, Loss: 2.278308629989624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6225 , TFLOPS: 102.85667792271059, Tokens per sec: 84046.83929708846, Loss: 2.271374464035034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6226 , TFLOPS: 101.45007175150744, Tokens per sec: 82897.46518533425, Loss: 2.272759437561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6227 , TFLOPS: 101.40052145357323, Tokens per sec: 82856.97636135406, Loss: 2.29500675201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6228 , TFLOPS: 102.02432850589926, Tokens per sec: 83366.70516203177, Loss: 2.2719509601593018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6229 , TFLOPS: 102.79759880713318, Tokens per sec: 83998.56423091843, Loss: 2.2783584594726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6230 , TFLOPS: 101.07569824498606, Tokens per sec: 82591.55495592422, Loss: 2.2666969299316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6231 , TFLOPS: 102.19373114230912, Tokens per sec: 83505.12841705407, Loss: 2.2861526012420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6232 , TFLOPS: 102.68540542427083, Tokens per sec: 83906.88812966697, Loss: 2.281803846359253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6233 , TFLOPS: 97.8058236592217, Tokens per sec: 79919.656257836, Loss: 2.282670497894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6234 , TFLOPS: 102.5960643897637, Tokens per sec: 83833.88527052838, Loss: 2.296675205230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6235 , TFLOPS: 101.55399506878516, Tokens per sec: 82982.38360310605, Loss: 2.2745659351348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6236 , TFLOPS: 100.8804236559346, Tokens per sec: 82431.99106239516, Loss: 2.2762057781219482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6237 , TFLOPS: 102.36995647852217, Tokens per sec: 83649.12667571749, Loss: 2.27280592918396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6238 , TFLOPS: 100.4251350413468, Tokens per sec: 82059.96301524388, Loss: 2.2938761711120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6239 , TFLOPS: 101.78993554942592, Tokens per sec: 83175.17664348592, Loss: 2.2634458541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6240 , TFLOPS: 101.09921320693073, Tokens per sec: 82610.76963665818, Loss: 2.2878668308258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6241 , TFLOPS: 102.07985857701931, Tokens per sec: 83412.08021261515, Loss: 2.292043685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6242 , TFLOPS: 101.66487042538289, Tokens per sec: 83072.98271116766, Loss: 2.289107322692871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6243 , TFLOPS: 101.63269142343106, Tokens per sec: 83046.6884203116, Loss: 2.2839622497558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6244 , TFLOPS: 103.38226049752787, Tokens per sec: 84476.30634867077, Loss: 2.2731447219848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6245 , TFLOPS: 101.0206725653186, Tokens per sec: 82546.59205658094, Loss: 2.2802536487579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6246 , TFLOPS: 101.76801177165194, Tokens per sec: 83157.26216029856, Loss: 2.29355525970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6247 , TFLOPS: 101.34652380397588, Tokens per sec: 82812.85349184491, Loss: 2.287782669067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6248 , TFLOPS: 103.42764636713025, Tokens per sec: 84513.3923110591, Loss: 2.278226613998413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6249 , TFLOPS: 103.27961272631933, Tokens per sec: 84392.43021242756, Loss: 2.275343656539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6250 , TFLOPS: 101.95349985196594, Tokens per sec: 83308.8292456111, Loss: 2.266237258911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6251 , TFLOPS: 103.20355057511948, Tokens per sec: 84330.2778706682, Loss: 2.272181987762451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6252 , TFLOPS: 101.96315391512047, Tokens per sec: 83316.71782913235, Loss: 2.2843003273010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6253 , TFLOPS: 102.7332389756425, Tokens per sec: 83945.97415583793, Loss: 2.314274549484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6254 , TFLOPS: 101.8911460877196, Tokens per sec: 83257.87837971673, Loss: 2.263932943344116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6255 , TFLOPS: 101.43671826549884, Tokens per sec: 82886.55370816731, Loss: 2.2929935455322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6256 , TFLOPS: 102.0130988116498, Tokens per sec: 83357.52908978249, Loss: 2.2746245861053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6257 , TFLOPS: 102.37851808053026, Tokens per sec: 83656.12258111352, Loss: 2.2963595390319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6258 , TFLOPS: 102.27406236339256, Tokens per sec: 83570.76912571095, Loss: 2.2820088863372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6259 , TFLOPS: 102.59404723310395, Tokens per sec: 83832.23699989547, Loss: 2.26255464553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6260 , TFLOPS: 100.66160312696663, Tokens per sec: 82253.18717523389, Loss: 2.281824827194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6261 , TFLOPS: 101.50637464219763, Tokens per sec: 82943.47172668255, Loss: 2.2580389976501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6262 , TFLOPS: 103.37648567845235, Tokens per sec: 84471.5875953471, Loss: 2.262505531311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6263 , TFLOPS: 101.61950763620908, Tokens per sec: 83035.91560839166, Loss: 2.28108286857605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6264 , TFLOPS: 102.19183223418766, Tokens per sec: 83503.57677034578, Loss: 2.266446828842163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6265 , TFLOPS: 101.48774300124042, Tokens per sec: 82928.24733323531, Loss: 2.3129940032958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6266 , TFLOPS: 101.49005295446129, Tokens per sec: 82930.13485547557, Loss: 2.2714743614196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6267 , TFLOPS: 101.99247389053436, Tokens per sec: 83340.67593580636, Loss: 2.270343780517578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6268 , TFLOPS: 100.66204838565122, Tokens per sec: 82253.55100756705, Loss: 2.275097608566284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6269 , TFLOPS: 101.76721400268417, Tokens per sec: 83156.61028273859, Loss: 2.2799978256225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6270 , TFLOPS: 102.0252714247007, Tokens per sec: 83367.47564525732, Loss: 2.3062567710876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6271 , TFLOPS: 99.94414429149447, Tokens per sec: 81666.93309173611, Loss: 2.288823127746582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6272 , TFLOPS: 101.85479508099851, Tokens per sec: 83228.17503636677, Loss: 2.2864341735839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6273 , TFLOPS: 101.51564680343101, Tokens per sec: 82951.04824831302, Loss: 2.279200315475464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6274 , TFLOPS: 101.54188954586039, Tokens per sec: 82972.49186870034, Loss: 2.296800136566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6275 , TFLOPS: 101.699178921337, Tokens per sec: 83101.01706639101, Loss: 2.283097982406616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6276 , TFLOPS: 100.4513063112864, Tokens per sec: 82081.34823361995, Loss: 2.293903112411499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6277 , TFLOPS: 102.32028455560818, Tokens per sec: 83608.53846883541, Loss: 2.3019495010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6278 , TFLOPS: 100.60546388615964, Tokens per sec: 82207.31435641788, Loss: 2.3033194541931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6279 , TFLOPS: 100.81320585266837, Tokens per sec: 82377.06566500613, Loss: 2.2782111167907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6280 , TFLOPS: 101.60524238873299, Tokens per sec: 83024.2591074588, Loss: 2.2974486351013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6281 , TFLOPS: 101.57776661876261, Tokens per sec: 83001.80794852677, Loss: 2.2816948890686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6282 , TFLOPS: 102.66200996062756, Tokens per sec: 83887.77109407126, Loss: 2.296152114868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6283 , TFLOPS: 101.46322700310621, Tokens per sec: 82908.21468006191, Loss: 2.283742904663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6284 , TFLOPS: 100.40608700247385, Tokens per sec: 82044.39837234064, Loss: 2.27348256111145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6285 , TFLOPS: 102.70048519470062, Tokens per sec: 83919.210198273, Loss: 2.2763493061065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6286 , TFLOPS: 103.39032350061699, Tokens per sec: 84482.89483605504, Loss: 2.2843422889709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6287 , TFLOPS: 102.07331546747248, Tokens per sec: 83406.73366936983, Loss: 2.2834107875823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6288 , TFLOPS: 102.02115093133962, Tokens per sec: 83364.10868406104, Loss: 2.2620928287506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6289 , TFLOPS: 101.95646900475077, Tokens per sec: 83311.25541678438, Loss: 2.2794928550720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6290 , TFLOPS: 101.99717345646499, Tokens per sec: 83344.51607210586, Loss: 2.271270751953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6291 , TFLOPS: 103.4264242308557, Tokens per sec: 84512.39367204858, Loss: 2.2802205085754395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6292 , TFLOPS: 100.83796603498972, Tokens per sec: 82397.2978473647, Loss: 2.2888245582580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6293 , TFLOPS: 102.22338301604191, Tokens per sec: 83529.35772638828, Loss: 2.2794675827026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6294 , TFLOPS: 101.47665009621313, Tokens per sec: 82919.18303498074, Loss: 2.2931597232818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6295 , TFLOPS: 102.73298304201987, Tokens per sec: 83945.76502588656, Loss: 2.2723562717437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6296 , TFLOPS: 102.05864780890994, Tokens per sec: 83394.74834797924, Loss: 2.272893190383911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6297 , TFLOPS: 102.2485675672496, Tokens per sec: 83549.9366714881, Loss: 2.3032279014587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6298 , TFLOPS: 100.23212503806985, Tokens per sec: 81902.24957304678, Loss: 2.286435842514038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6299 , TFLOPS: 103.28187624331623, Tokens per sec: 84394.27979043384, Loss: 2.2686991691589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6300 , TFLOPS: 101.64252135109007, Tokens per sec: 83054.72070724664, Loss: 2.2739980220794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6301 , TFLOPS: 101.60948266243152, Tokens per sec: 83027.72394425223, Loss: 2.2811288833618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6302 , TFLOPS: 103.04019394638213, Tokens per sec: 84196.79496415319, Loss: 2.270970344543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6303 , TFLOPS: 99.65478204918581, Tokens per sec: 81430.48775469892, Loss: 2.281271457672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6304 , TFLOPS: 101.4969867688607, Tokens per sec: 82935.8006537136, Loss: 2.2860360145568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6305 , TFLOPS: 102.7072098462546, Tokens per sec: 83924.70508416569, Loss: 2.287341833114624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6306 , TFLOPS: 101.55746113334091, Tokens per sec: 82985.21581367901, Loss: 2.2780253887176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6307 , TFLOPS: 101.60354046336967, Tokens per sec: 83022.86842043298, Loss: 2.2768657207489014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6308 , TFLOPS: 102.31287160512771, Tokens per sec: 83602.4811561715, Loss: 2.2831289768218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6309 , TFLOPS: 99.65889447585064, Tokens per sec: 81433.84812439005, Loss: 2.29990291595459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6310 , TFLOPS: 102.73973914711232, Tokens per sec: 83951.28561327522, Loss: 2.278550624847412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6311 , TFLOPS: 101.51253055683532, Tokens per sec: 82948.50188300034, Loss: 2.270305633544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6312 , TFLOPS: 100.94027221658048, Tokens per sec: 82480.89486193791, Loss: 2.282702922821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6313 , TFLOPS: 100.82202836394414, Tokens per sec: 82384.2747660812, Loss: 2.2902090549468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6314 , TFLOPS: 101.01851805110542, Tokens per sec: 82544.83154755521, Loss: 2.2794339656829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6315 , TFLOPS: 102.31223732478277, Tokens per sec: 83601.96286937383, Loss: 2.2914469242095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6316 , TFLOPS: 100.63957559145985, Tokens per sec: 82235.18790893209, Loss: 2.2709405422210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6317 , TFLOPS: 100.84572689583533, Tokens per sec: 82403.63944654398, Loss: 2.286714792251587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6318 , TFLOPS: 102.34123806801472, Tokens per sec: 83625.6601232142, Loss: 2.2810654640197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6319 , TFLOPS: 102.6920462056065, Tokens per sec: 83912.31448304528, Loss: 2.28706955909729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6320 , TFLOPS: 103.32900380512152, Tokens per sec: 84432.7889343563, Loss: 2.292525053024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6321 , TFLOPS: 102.08417846752471, Tokens per sec: 83415.610106351, Loss: 2.2792537212371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6322 , TFLOPS: 102.30296988878258, Tokens per sec: 83594.39020885304, Loss: 2.2807607650756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6323 , TFLOPS: 102.1122957928565, Tokens per sec: 83438.58549668394, Loss: 2.294447183609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6324 , TFLOPS: 102.70490690822494, Tokens per sec: 83922.82329421876, Loss: 2.2802557945251465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6325 , TFLOPS: 102.11811062818768, Tokens per sec: 83443.33694832024, Loss: 2.283719062805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6326 , TFLOPS: 101.33290799123799, Tokens per sec: 82801.72766075458, Loss: 2.2891359329223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6327 , TFLOPS: 102.72250752282523, Tokens per sec: 83937.20520948878, Loss: 2.288482189178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6328 , TFLOPS: 102.11779734316303, Tokens per sec: 83443.08095506192, Loss: 2.292581796646118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6329 , TFLOPS: 103.26609057937152, Tokens per sec: 84381.3809180655, Loss: 2.272994041442871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6330 , TFLOPS: 100.63883507393257, Tokens per sec: 82234.58281299792, Loss: 2.25642466545105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6331 , TFLOPS: 102.20203510122874, Tokens per sec: 83511.91379565021, Loss: 2.270277976989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6332 , TFLOPS: 101.44342682963962, Tokens per sec: 82892.03544862055, Loss: 2.2863755226135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6333 , TFLOPS: 102.68316324131676, Tokens per sec: 83905.0559842563, Loss: 2.2775521278381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6334 , TFLOPS: 101.47532458240315, Tokens per sec: 82918.09992352483, Loss: 2.2799010276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6335 , TFLOPS: 101.31030868160936, Tokens per sec: 82783.26118309904, Loss: 2.2670371532440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6336 , TFLOPS: 100.25121893501952, Tokens per sec: 81917.851687765, Loss: 2.2651631832122803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6337 , TFLOPS: 103.2878138477657, Tokens per sec: 84399.13155987705, Loss: 2.2956950664520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6338 , TFLOPS: 101.5620599287324, Tokens per sec: 82988.9736077769, Loss: 2.2809641361236572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6339 , TFLOPS: 101.69322440435008, Tokens per sec: 83096.15147727811, Loss: 2.2952346801757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6340 , TFLOPS: 102.88007439371532, Tokens per sec: 84065.95715582516, Loss: 2.2960524559020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6341 , TFLOPS: 100.3535745313938, Tokens per sec: 82001.48908042986, Loss: 2.273651123046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6342 , TFLOPS: 101.49526556385116, Tokens per sec: 82934.39421279254, Loss: 2.290210008621216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6343 , TFLOPS: 102.71231288034987, Tokens per sec: 83928.87490468871, Loss: 2.279372215270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6344 , TFLOPS: 100.82539904518467, Tokens per sec: 82387.02903649189, Loss: 2.271817684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6345 , TFLOPS: 101.48722731284816, Tokens per sec: 82927.82595097499, Loss: 2.2861104011535645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6346 , TFLOPS: 102.26744449831844, Tokens per sec: 83565.36149779988, Loss: 2.2845120429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6347 , TFLOPS: 100.10218686684394, Tokens per sec: 81796.07374843198, Loss: 2.3023765087127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6348 , TFLOPS: 102.74916082604274, Tokens per sec: 83958.98430966487, Loss: 2.2851715087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6349 , TFLOPS: 100.51620905458587, Tokens per sec: 82134.38193591518, Loss: 2.274970531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6350 , TFLOPS: 101.34979107968334, Tokens per sec: 82815.52326693264, Loss: 2.286337375640869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6351 , TFLOPS: 100.78244260316605, Tokens per sec: 82351.92822192128, Loss: 2.312518835067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6352 , TFLOPS: 101.69418967511379, Tokens per sec: 83096.94022486742, Loss: 2.2594826221466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6353 , TFLOPS: 102.24088229159476, Tokens per sec: 83543.65683491393, Loss: 2.256077527999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6354 , TFLOPS: 100.10130596925521, Tokens per sec: 81795.3539443359, Loss: 2.2737274169921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6355 , TFLOPS: 101.37136596287745, Tokens per sec: 82833.15265937743, Loss: 2.2863805294036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6356 , TFLOPS: 102.32748220730464, Tokens per sec: 83614.41985532064, Loss: 2.2854130268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6357 , TFLOPS: 102.06214365222674, Tokens per sec: 83397.60489154443, Loss: 2.2728431224823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6358 , TFLOPS: 103.24999266325875, Tokens per sec: 84368.22689641251, Loss: 2.267249822616577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6359 , TFLOPS: 102.01786076166464, Tokens per sec: 83361.42020172311, Loss: 2.2985002994537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6360 , TFLOPS: 101.67296878231511, Tokens per sec: 83079.60008708722, Loss: 2.2914674282073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6361 , TFLOPS: 103.44274250176191, Tokens per sec: 84525.72775127571, Loss: 2.280669689178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6362 , TFLOPS: 100.7324244056784, Tokens per sec: 82311.05706517119, Loss: 2.2985270023345947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6363 , TFLOPS: 102.39811578506006, Tokens per sec: 83672.13636997463, Loss: 2.2733705043792725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6364 , TFLOPS: 102.11795950266674, Tokens per sec: 83443.21345976676, Loss: 2.2724692821502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6365 , TFLOPS: 102.17615329900934, Tokens per sec: 83490.76510879988, Loss: 2.2824113368988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6366 , TFLOPS: 102.16943546405966, Tokens per sec: 83485.27579293004, Loss: 2.292429208755493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6367 , TFLOPS: 101.45432273158251, Tokens per sec: 82900.93877058355, Loss: 2.2885711193084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6368 , TFLOPS: 102.11138146756436, Tokens per sec: 83437.83837795092, Loss: 2.289663314819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6369 , TFLOPS: 102.86500997686717, Tokens per sec: 84053.64763302596, Loss: 2.2998857498168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6370 , TFLOPS: 99.68602600125102, Tokens per sec: 81456.01799222229, Loss: 2.2698941230773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6371 , TFLOPS: 102.83581883717723, Tokens per sec: 84029.79480133852, Loss: 2.2758278846740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6372 , TFLOPS: 101.96860417200308, Tokens per sec: 83321.17137433324, Loss: 2.291533946990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6373 , TFLOPS: 99.71677570146241, Tokens per sec: 81481.14436383272, Loss: 2.274122476577759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6374 , TFLOPS: 102.16748751769565, Tokens per sec: 83483.68407581141, Loss: 2.27850604057312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6375 , TFLOPS: 101.09771657406995, Tokens per sec: 82609.54669942087, Loss: 2.264068365097046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6376 , TFLOPS: 102.23494575830216, Tokens per sec: 83538.8059407406, Loss: 2.2750134468078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6377 , TFLOPS: 101.0595258085167, Tokens per sec: 82578.34004176955, Loss: 2.28550386428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6378 , TFLOPS: 102.16698330893486, Tokens per sec: 83483.27207385348, Loss: 2.2784292697906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6379 , TFLOPS: 101.6429876840657, Tokens per sec: 83055.1017599255, Loss: 2.3059051036834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6380 , TFLOPS: 101.54107352343814, Tokens per sec: 82971.82507577284, Loss: 2.264007091522217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6381 , TFLOPS: 102.21631825659944, Tokens per sec: 83523.58492958474, Loss: 2.2890737056732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6382 , TFLOPS: 101.8817395401083, Tokens per sec: 83250.19204751746, Loss: 2.2847044467926025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6383 , TFLOPS: 101.16870736020188, Tokens per sec: 82667.55509823469, Loss: 2.2903871536254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6384 , TFLOPS: 102.21174931183181, Tokens per sec: 83519.85152720001, Loss: 2.26255464553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6385 , TFLOPS: 101.09447526908565, Tokens per sec: 82606.89814567959, Loss: 2.2776503562927246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6386 , TFLOPS: 100.85627625882498, Tokens per sec: 82412.25960260758, Loss: 2.297123670578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6387 , TFLOPS: 101.56755240304675, Tokens per sec: 82993.46164992804, Loss: 2.2814221382141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6388 , TFLOPS: 102.06492947362746, Tokens per sec: 83399.88125792438, Loss: 2.2851104736328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6389 , TFLOPS: 101.41550581531163, Tokens per sec: 82869.2204690623, Loss: 2.289641857147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6390 , TFLOPS: 102.86268135375376, Tokens per sec: 84051.74485513588, Loss: 2.2861647605895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6391 , TFLOPS: 102.3186019775007, Tokens per sec: 83607.16359095038, Loss: 2.2897653579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6392 , TFLOPS: 100.57714763547752, Tokens per sec: 82184.17641906037, Loss: 2.290773868560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6393 , TFLOPS: 102.30841729150394, Tokens per sec: 83598.84142184528, Loss: 2.2933788299560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6394 , TFLOPS: 101.5022899231694, Tokens per sec: 82940.13399761457, Loss: 2.2997055053710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6395 , TFLOPS: 102.76110130449574, Tokens per sec: 83968.74118198406, Loss: 2.3003926277160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6396 , TFLOPS: 103.36767723221823, Tokens per sec: 84464.3899871796, Loss: 2.2843174934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6397 , TFLOPS: 102.64062386922959, Tokens per sec: 83870.29596826302, Loss: 2.27768611907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6398 , TFLOPS: 101.69305423247431, Tokens per sec: 83096.0124254578, Loss: 2.308213233947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6399 , TFLOPS: 102.948143781517, Tokens per sec: 84121.57840485984, Loss: 2.271904230117798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6400 , TFLOPS: 101.92496998360492, Tokens per sec: 83285.51675574917, Loss: 2.3001081943511963 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/model.pt +[rank0]:[2024-08-28 02:29:45,422] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007680730981519446, 'preprocessing_with_comm': 0.0015088299987837672, 'state_converting': 3.368229029001668, : 3.379066136025358}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400, took 15.89s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004400 + eval ppl=7.824367046356201, eval loss=2.0572428703308105 +------------------------------------------------------------------ +iteration: 6401 , TFLOPS: 99.34337638334738, Tokens per sec: 81176.03016884736, Loss: 2.279832601547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6402 , TFLOPS: 100.14802375247456, Tokens per sec: 81833.52824762717, Loss: 2.2680020332336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6403 , TFLOPS: 102.37028487531396, Tokens per sec: 83649.39501718973, Loss: 2.270458221435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6404 , TFLOPS: 100.92260163083526, Tokens per sec: 82466.45577144415, Loss: 2.2809391021728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6405 , TFLOPS: 102.17179822340955, Tokens per sec: 83487.20646441769, Loss: 2.3039236068725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6406 , TFLOPS: 102.04739123777998, Tokens per sec: 83385.55031393903, Loss: 2.280919313430786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6407 , TFLOPS: 102.66891527747252, Tokens per sec: 83893.41360622396, Loss: 2.3067471981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6408 , TFLOPS: 103.33001468989907, Tokens per sec: 84433.61495433054, Loss: 2.272249937057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6409 , TFLOPS: 103.33150436449684, Tokens per sec: 84434.832205792, Loss: 2.2723071575164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6410 , TFLOPS: 102.70527675702799, Tokens per sec: 83923.12550719647, Loss: 2.2757973670959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6411 , TFLOPS: 101.3143420672039, Tokens per sec: 82786.55696629692, Loss: 2.259350538253784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6412 , TFLOPS: 102.59152676886208, Tokens per sec: 83830.17746368091, Loss: 2.2759807109832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6413 , TFLOPS: 102.5524805975863, Tokens per sec: 83798.27183199361, Loss: 2.2836146354675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6414 , TFLOPS: 101.83917621443192, Tokens per sec: 83215.41245842981, Loss: 2.3014965057373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6415 , TFLOPS: 103.31949000646385, Tokens per sec: 84425.01496456617, Loss: 2.2776215076446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6416 , TFLOPS: 101.86363095841574, Tokens per sec: 83235.39505926009, Loss: 2.2831897735595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6417 , TFLOPS: 99.47488118284643, Tokens per sec: 81283.48612575272, Loss: 2.267622709274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6418 , TFLOPS: 103.33720315603851, Tokens per sec: 84439.48883506066, Loss: 2.2910921573638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6419 , TFLOPS: 102.70140970595455, Tokens per sec: 83919.96564021762, Loss: 2.306509256362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6420 , TFLOPS: 102.04263823638424, Tokens per sec: 83381.66651414543, Loss: 2.29447078704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6421 , TFLOPS: 100.9057310693732, Tokens per sec: 82452.67040138663, Loss: 2.267918586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6422 , TFLOPS: 102.5740872606508, Tokens per sec: 83815.9271925886, Loss: 2.2647130489349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6423 , TFLOPS: 101.90602923633689, Tokens per sec: 83270.03978357825, Loss: 2.2674174308776855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6424 , TFLOPS: 100.21777052840172, Tokens per sec: 81890.52013367899, Loss: 2.282555103302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6425 , TFLOPS: 101.61782152957522, Tokens per sec: 83034.53784725688, Loss: 2.287055492401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6426 , TFLOPS: 102.53247098491713, Tokens per sec: 83781.92146238837, Loss: 2.2908473014831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6427 , TFLOPS: 99.23822538923059, Tokens per sec: 81090.10858472712, Loss: 2.2864253520965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6428 , TFLOPS: 102.72643204073621, Tokens per sec: 83940.41203409992, Loss: 2.2709500789642334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6429 , TFLOPS: 102.14689019521973, Tokens per sec: 83466.85347339409, Loss: 2.2585666179656982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6430 , TFLOPS: 99.71957435978635, Tokens per sec: 81483.43121959308, Loss: 2.2718582153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6431 , TFLOPS: 102.03449928071699, Tokens per sec: 83375.01596395437, Loss: 2.270190715789795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6432 , TFLOPS: 100.80272361232605, Tokens per sec: 82368.50035658554, Loss: 2.260326623916626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6433 , TFLOPS: 102.25369079292165, Tokens per sec: 83554.12299106829, Loss: 2.2783143520355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6434 , TFLOPS: 99.80117462939721, Tokens per sec: 81550.10890046992, Loss: 2.2643489837646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6435 , TFLOPS: 102.34888127451683, Tokens per sec: 83631.9055840007, Loss: 2.2661991119384766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6436 , TFLOPS: 101.02138560450551, Tokens per sec: 82547.1746992558, Loss: 2.2747535705566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6437 , TFLOPS: 100.82271089120637, Tokens per sec: 82384.83247667698, Loss: 2.2784194946289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6438 , TFLOPS: 101.53886231095971, Tokens per sec: 82970.01823713505, Loss: 2.2924203872680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6439 , TFLOPS: 101.150732668955, Tokens per sec: 82652.86751530715, Loss: 2.280928611755371 +------------------------------------------------------------------ +wandb: 429 encountered (Filestream rate limit exceeded, retrying in 2.2 seconds.), retrying request +------------------------------------------------------------------ +iteration: 6440 , TFLOPS: 101.57492243597852, Tokens per sec: 82999.4838935581, Loss: 2.2812070846557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6441 , TFLOPS: 102.22370280663955, Tokens per sec: 83529.61903551775, Loss: 2.2747092247009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6442 , TFLOPS: 102.04164167039578, Tokens per sec: 83380.85219442238, Loss: 2.2679178714752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6443 , TFLOPS: 102.64720420296992, Tokens per sec: 83875.6729283551, Loss: 2.278931140899658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6444 , TFLOPS: 101.33166071329794, Tokens per sec: 82800.70847784197, Loss: 2.294130563735962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6445 , TFLOPS: 102.59695650494096, Tokens per sec: 83834.61424080482, Loss: 2.2799525260925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6446 , TFLOPS: 102.63890705366752, Tokens per sec: 83868.89311407249, Loss: 2.26572322845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6447 , TFLOPS: 102.80359522503113, Tokens per sec: 84003.46405834437, Loss: 2.288651704788208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6448 , TFLOPS: 102.04461780754902, Tokens per sec: 83383.28407270287, Loss: 2.2805299758911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6449 , TFLOPS: 100.63908045269818, Tokens per sec: 82234.78331830393, Loss: 2.278348684310913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6450 , TFLOPS: 103.36222548617525, Tokens per sec: 84459.93522514746, Loss: 2.2957048416137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6451 , TFLOPS: 102.62594216600094, Tokens per sec: 83858.29917061422, Loss: 2.3025312423706055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6452 , TFLOPS: 101.50672858740059, Tokens per sec: 82943.76094441925, Loss: 2.252415180206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6453 , TFLOPS: 103.29355426361938, Tokens per sec: 84403.82220144254, Loss: 2.2658917903900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6454 , TFLOPS: 102.71305701732798, Tokens per sec: 83929.48295816932, Loss: 2.294348955154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6455 , TFLOPS: 100.2717206376994, Tokens per sec: 81934.6041567855, Loss: 2.2953598499298096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6456 , TFLOPS: 103.32633078344445, Tokens per sec: 84430.60473953429, Loss: 2.291478157043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6457 , TFLOPS: 101.91468718394529, Tokens per sec: 83277.11441544446, Loss: 2.2872891426086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6458 , TFLOPS: 101.33672714179669, Tokens per sec: 82804.84838698978, Loss: 2.292152166366577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6459 , TFLOPS: 100.47777813462888, Tokens per sec: 82102.97904191843, Loss: 2.2740590572357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6460 , TFLOPS: 102.77214644461037, Tokens per sec: 83977.76645029894, Loss: 2.2846882343292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6461 , TFLOPS: 102.83277266787175, Tokens per sec: 84027.30569798363, Loss: 2.263615608215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6462 , TFLOPS: 99.60241286749485, Tokens per sec: 81387.69554823656, Loss: 2.262775182723999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6463 , TFLOPS: 101.43325067181135, Tokens per sec: 82883.72024810147, Loss: 2.270855665206909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6464 , TFLOPS: 102.50330035519876, Tokens per sec: 83758.08538992668, Loss: 2.28330135345459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6465 , TFLOPS: 99.88656040285899, Tokens per sec: 81619.87981399089, Loss: 2.2723779678344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6466 , TFLOPS: 101.68354858664746, Tokens per sec: 83088.24511755559, Loss: 2.304171323776245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6467 , TFLOPS: 102.89824716904103, Tokens per sec: 84080.80659835266, Loss: 2.2919068336486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6468 , TFLOPS: 99.38933412347849, Tokens per sec: 81213.58342136595, Loss: 2.2941951751708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6469 , TFLOPS: 102.63236524293131, Tokens per sec: 83863.54763211873, Loss: 2.2824249267578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6470 , TFLOPS: 101.03339362039246, Tokens per sec: 82556.98675815087, Loss: 2.2828869819641113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6471 , TFLOPS: 102.8689813889456, Tokens per sec: 84056.89277606847, Loss: 2.2959227561950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6472 , TFLOPS: 100.40379344851475, Tokens per sec: 82042.52425035933, Loss: 2.305248260498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6473 , TFLOPS: 102.21996217631703, Tokens per sec: 83526.56247018883, Loss: 2.288595199584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6474 , TFLOPS: 101.3496175836036, Tokens per sec: 82815.3814988197, Loss: 2.278273582458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6475 , TFLOPS: 101.0158714231266, Tokens per sec: 82542.66891970344, Loss: 2.2731707096099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6476 , TFLOPS: 100.9740867336984, Tokens per sec: 82508.52557433807, Loss: 2.291090726852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6477 , TFLOPS: 100.47679729934107, Tokens per sec: 82102.17757615582, Loss: 2.290248155593872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6478 , TFLOPS: 102.184018756277, Tokens per sec: 83497.19217640818, Loss: 2.287893772125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6479 , TFLOPS: 101.4772693829036, Tokens per sec: 82919.68907007737, Loss: 2.266929864883423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6480 , TFLOPS: 102.80997502532621, Tokens per sec: 84008.67715739615, Loss: 2.2495875358581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6481 , TFLOPS: 100.49065830612129, Tokens per sec: 82113.50376161, Loss: 2.2948572635650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6482 , TFLOPS: 101.30578558687951, Tokens per sec: 82779.56524596046, Loss: 2.2562742233276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6483 , TFLOPS: 102.62360190854437, Tokens per sec: 83856.38688600287, Loss: 2.271716594696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6484 , TFLOPS: 102.66114280540333, Tokens per sec: 83887.06251921497, Loss: 2.2699332237243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6485 , TFLOPS: 103.2410622314761, Tokens per sec: 84360.92961071315, Loss: 2.286961793899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6486 , TFLOPS: 102.13565666736892, Tokens per sec: 83457.67426861046, Loss: 2.298330307006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6487 , TFLOPS: 100.24704060550604, Tokens per sec: 81914.43746715975, Loss: 2.285734176635742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6488 , TFLOPS: 103.18686571335559, Tokens per sec: 84316.64424061439, Loss: 2.2802629470825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6489 , TFLOPS: 103.25906807202924, Tokens per sec: 84375.64264654068, Loss: 2.289041519165039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6490 , TFLOPS: 101.42844862005892, Tokens per sec: 82879.79636799874, Loss: 2.2824792861938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6491 , TFLOPS: 102.75537330455519, Tokens per sec: 83964.06068578082, Loss: 2.2888360023498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6492 , TFLOPS: 102.72755521833008, Tokens per sec: 83941.32981142502, Loss: 2.289024829864502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6493 , TFLOPS: 99.99431179374697, Tokens per sec: 81707.92624925303, Loss: 2.2755446434020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6494 , TFLOPS: 102.53872263265234, Tokens per sec: 83787.02984468456, Loss: 2.270082950592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6495 , TFLOPS: 102.03630216245004, Tokens per sec: 83376.48914502877, Loss: 2.259290933609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6496 , TFLOPS: 101.97612296484414, Tokens per sec: 83327.31517351461, Loss: 2.2905354499816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6497 , TFLOPS: 101.1492397545678, Tokens per sec: 82651.64761653033, Loss: 2.2877180576324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6498 , TFLOPS: 102.18542367467211, Tokens per sec: 83498.34017139464, Loss: 2.290848731994629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6499 , TFLOPS: 101.9926090700592, Tokens per sec: 83340.78639447594, Loss: 2.291541814804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6500 , TFLOPS: 100.48735897232498, Tokens per sec: 82110.80779103262, Loss: 2.289045810699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6501 , TFLOPS: 101.42127493866809, Tokens per sec: 82873.93456826732, Loss: 2.299316883087158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6502 , TFLOPS: 102.66514074015807, Tokens per sec: 83890.32933461906, Loss: 2.289497137069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6503 , TFLOPS: 99.89026798778443, Tokens per sec: 81622.90937707521, Loss: 2.2834930419921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6504 , TFLOPS: 101.6195818810127, Tokens per sec: 83035.97627573194, Loss: 2.28350830078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6505 , TFLOPS: 102.37536531184675, Tokens per sec: 83653.54637266276, Loss: 2.282036781311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6506 , TFLOPS: 99.88840725389711, Tokens per sec: 81621.38892351632, Loss: 2.2913239002227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6507 , TFLOPS: 102.78936433340957, Tokens per sec: 83991.83563046464, Loss: 2.2843985557556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6508 , TFLOPS: 100.81836031941866, Tokens per sec: 82381.27751247557, Loss: 2.2876460552215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6509 , TFLOPS: 102.82916154266883, Tokens per sec: 84024.3549546217, Loss: 2.2780165672302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6510 , TFLOPS: 101.62422913516883, Tokens per sec: 83039.77366673388, Loss: 2.298931837081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6511 , TFLOPS: 102.0559387904108, Tokens per sec: 83392.53473922727, Loss: 2.2695751190185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6512 , TFLOPS: 101.47060393652664, Tokens per sec: 82914.2425622589, Loss: 2.2927045822143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6513 , TFLOPS: 102.25454346574828, Tokens per sec: 83554.8197319847, Loss: 2.3037145137786865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6514 , TFLOPS: 100.25402988339187, Tokens per sec: 81920.14858604028, Loss: 2.291489362716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6515 , TFLOPS: 100.57806061049052, Tokens per sec: 82184.92243444557, Loss: 2.2684996128082275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6516 , TFLOPS: 101.56121153211656, Tokens per sec: 82988.28036106244, Loss: 2.2839858531951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6517 , TFLOPS: 102.14102457735424, Tokens per sec: 83462.06052604177, Loss: 2.2532148361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6518 , TFLOPS: 101.94653372193153, Tokens per sec: 83303.13704143584, Loss: 2.3108251094818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6519 , TFLOPS: 100.9533414190312, Tokens per sec: 82491.5740436916, Loss: 2.3055522441864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6520 , TFLOPS: 102.55069980259915, Tokens per sec: 83796.81669856787, Loss: 2.27742862701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6521 , TFLOPS: 103.20099434996914, Tokens per sec: 84328.18911329476, Loss: 2.2700533866882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6522 , TFLOPS: 101.42698595864151, Tokens per sec: 82878.60118970233, Loss: 2.276221752166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6523 , TFLOPS: 102.90789645001438, Tokens per sec: 84088.69127423057, Loss: 2.271453380584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6524 , TFLOPS: 102.76326634347541, Tokens per sec: 83970.51029106742, Loss: 2.267925500869751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6525 , TFLOPS: 99.47703997282137, Tokens per sec: 81285.25012861344, Loss: 2.2819693088531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6526 , TFLOPS: 102.74335675536523, Tokens per sec: 83954.2416541042, Loss: 2.2843616008758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6527 , TFLOPS: 102.80455744096379, Tokens per sec: 84004.25030975258, Loss: 2.255058526992798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6528 , TFLOPS: 101.21502444725874, Tokens per sec: 82705.40198237674, Loss: 2.298581838607788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6529 , TFLOPS: 103.361843917538, Tokens per sec: 84459.62343559152, Loss: 2.2933285236358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6530 , TFLOPS: 100.90476059994631, Tokens per sec: 82451.8774058358, Loss: 2.2893407344818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6531 , TFLOPS: 102.03510334240029, Tokens per sec: 83375.50955830573, Loss: 2.2830982208251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6532 , TFLOPS: 101.30751638956608, Tokens per sec: 82780.97952939048, Loss: 2.2774782180786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6533 , TFLOPS: 102.50371071272457, Tokens per sec: 83758.42070362456, Loss: 2.292790651321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6534 , TFLOPS: 100.75625871535914, Tokens per sec: 82330.53269316102, Loss: 2.275918483734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6535 , TFLOPS: 100.44016451505601, Tokens per sec: 82072.24398510474, Loss: 2.274630546569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6536 , TFLOPS: 102.76096435659487, Tokens per sec: 83968.62927832892, Loss: 2.2677578926086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6537 , TFLOPS: 101.88365183821035, Tokens per sec: 83251.75463552358, Loss: 2.2843832969665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6538 , TFLOPS: 100.50449565190127, Tokens per sec: 82124.81061305178, Loss: 2.282923936843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6539 , TFLOPS: 101.54094939516777, Tokens per sec: 82971.72364736776, Loss: 2.2724711894989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6540 , TFLOPS: 103.43075209468093, Tokens per sec: 84515.93008098929, Loss: 2.2770230770111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6541 , TFLOPS: 101.03921034298361, Tokens per sec: 82561.73975191585, Loss: 2.2748541831970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6542 , TFLOPS: 103.30622844235295, Tokens per sec: 84414.17859914812, Loss: 2.2783021926879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6543 , TFLOPS: 102.65476971463438, Tokens per sec: 83881.85490268972, Loss: 2.2753746509552 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6544 , TFLOPS: 101.08101053039499, Tokens per sec: 82595.89576108225, Loss: 2.2928762435913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6545 , TFLOPS: 103.3883865677739, Tokens per sec: 84481.31211836789, Loss: 2.272217035293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6546 , TFLOPS: 100.15305453468403, Tokens per sec: 81837.63902927437, Loss: 2.281416654586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6547 , TFLOPS: 102.01684187634963, Tokens per sec: 83360.58764430379, Loss: 2.265087127685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6548 , TFLOPS: 101.46453079154814, Tokens per sec: 82909.28003916031, Loss: 2.282588243484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6549 , TFLOPS: 101.99906794949511, Tokens per sec: 83346.06411112937, Loss: 2.270650625228882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6550 , TFLOPS: 101.56065319928634, Tokens per sec: 82987.8241329341, Loss: 2.291660785675049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6551 , TFLOPS: 100.3840209367476, Tokens per sec: 82026.36762200465, Loss: 2.2753212451934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6552 , TFLOPS: 102.30445240332647, Tokens per sec: 83595.60160965017, Loss: 2.279754400253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6553 , TFLOPS: 100.56539894467747, Tokens per sec: 82174.57626136944, Loss: 2.298421621322632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6554 , TFLOPS: 101.47875880983193, Tokens per sec: 82920.90611916177, Loss: 2.2881078720092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6555 , TFLOPS: 100.88659734245127, Tokens per sec: 82437.03574057276, Loss: 2.2957444190979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6556 , TFLOPS: 102.72799258560123, Tokens per sec: 83941.68719548112, Loss: 2.2818591594696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6557 , TFLOPS: 101.47099058688161, Tokens per sec: 82914.5585042172, Loss: 2.2708070278167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6558 , TFLOPS: 102.67078169419835, Tokens per sec: 83894.93870337647, Loss: 2.275862693786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6559 , TFLOPS: 102.74658220638392, Tokens per sec: 83956.87725316206, Loss: 2.294846534729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6560 , TFLOPS: 101.47072087083049, Tokens per sec: 82914.33811228874, Loss: 2.2634406089782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6561 , TFLOPS: 103.3986889442008, Tokens per sec: 84489.73045535306, Loss: 2.2820253372192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6562 , TFLOPS: 102.25703136539371, Tokens per sec: 83556.8526588292, Loss: 2.2748138904571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6563 , TFLOPS: 101.25501905554526, Tokens per sec: 82738.08260636048, Loss: 2.2596771717071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6564 , TFLOPS: 101.99368237351301, Tokens per sec: 83341.66341835733, Loss: 2.285661458969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6565 , TFLOPS: 103.39679543663523, Tokens per sec: 84488.18322157802, Loss: 2.2724502086639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6566 , TFLOPS: 101.51570866951047, Tokens per sec: 82951.09880067912, Loss: 2.2808756828308105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6567 , TFLOPS: 102.75136654418836, Tokens per sec: 83960.78665874188, Loss: 2.277290105819702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6568 , TFLOPS: 102.10372879195086, Tokens per sec: 83431.58517970973, Loss: 2.279268264770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6569 , TFLOPS: 100.13046284272379, Tokens per sec: 81819.17877621179, Loss: 2.2827792167663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6570 , TFLOPS: 102.84406204035061, Tokens per sec: 84036.5305348504, Loss: 2.2808847427368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6571 , TFLOPS: 102.79624491070587, Tokens per sec: 83997.45792729528, Loss: 2.2825002670288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6572 , TFLOPS: 102.82559760946819, Tokens per sec: 84021.44277305965, Loss: 2.279226064682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6573 , TFLOPS: 101.7470211693975, Tokens per sec: 83140.11019885028, Loss: 2.2794198989868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6574 , TFLOPS: 103.17986483740643, Tokens per sec: 84310.92364465774, Loss: 2.2652504444122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6575 , TFLOPS: 103.38776669632932, Tokens per sec: 84480.8056054536, Loss: 2.310548782348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6576 , TFLOPS: 100.91207148305067, Tokens per sec: 82457.85131661924, Loss: 2.2809057235717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6577 , TFLOPS: 102.69953230910944, Tokens per sec: 83918.43157092699, Loss: 2.2643096446990967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6578 , TFLOPS: 103.37421445849493, Tokens per sec: 84469.73172305348, Loss: 2.281503915786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6579 , TFLOPS: 99.24728725384654, Tokens per sec: 81097.51326757796, Loss: 2.2956793308258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6580 , TFLOPS: 102.52430054006805, Tokens per sec: 83775.24518157623, Loss: 2.2799267768859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6581 , TFLOPS: 103.40487507856936, Tokens per sec: 84494.78530499087, Loss: 2.271906614303589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6582 , TFLOPS: 100.88362902626035, Tokens per sec: 82434.6102530019, Loss: 2.2686195373535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6583 , TFLOPS: 102.98473693829851, Tokens per sec: 84151.47961525767, Loss: 2.2806217670440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6584 , TFLOPS: 100.30635525332171, Tokens per sec: 81962.90499278481, Loss: 2.2679824829101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6585 , TFLOPS: 102.79119502975365, Tokens per sec: 83993.33153957372, Loss: 2.279304027557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6586 , TFLOPS: 101.6210212244718, Tokens per sec: 83037.15240032435, Loss: 2.2762155532836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6587 , TFLOPS: 101.31791756154706, Tokens per sec: 82789.47859476613, Loss: 2.2949447631835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6588 , TFLOPS: 101.53963732328921, Tokens per sec: 82970.65151966007, Loss: 2.284651041030884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6589 , TFLOPS: 101.71405559400114, Tokens per sec: 83113.1731785845, Loss: 2.2716639041900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6590 , TFLOPS: 100.74388213159017, Tokens per sec: 82320.4194679618, Loss: 2.2661426067352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6591 , TFLOPS: 100.98262466675092, Tokens per sec: 82515.50213922125, Loss: 2.2815301418304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6592 , TFLOPS: 101.05411695822106, Tokens per sec: 82573.9203309568, Loss: 2.295870542526245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6593 , TFLOPS: 101.47895840285847, Tokens per sec: 82921.06921176173, Loss: 2.259286403656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6594 , TFLOPS: 102.26183600098518, Tokens per sec: 83560.77865025331, Loss: 2.3092682361602783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6595 , TFLOPS: 101.51949091628384, Tokens per sec: 82954.18937188132, Loss: 2.2891342639923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6596 , TFLOPS: 102.64919112385354, Tokens per sec: 83877.29649255698, Loss: 2.2684860229492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6597 , TFLOPS: 102.55814595416545, Tokens per sec: 83802.90114069369, Loss: 2.277642011642456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6598 , TFLOPS: 101.94571858108431, Tokens per sec: 83302.470968866, Loss: 2.287977933883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6599 , TFLOPS: 102.2267077584148, Tokens per sec: 83532.07445896765, Loss: 2.3079793453216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6600 , TFLOPS: 101.59309688927299, Tokens per sec: 83014.33470719706, Loss: 2.287813901901245 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/model.pt +[rank0]:[2024-08-28 03:22:41,149] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00775246397824958, 'preprocessing_with_comm': 0.0017262030160054564, 'state_converting': 3.422616792988265, : 3.433748502022354}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600, took 15.99s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004600 + eval ppl=8.009254455566406, eval loss=2.0805976390838623 +------------------------------------------------------------------ +iteration: 6601 , TFLOPS: 100.96227902515605, Tokens per sec: 82498.87719173067, Loss: 2.26379132270813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6602 , TFLOPS: 101.48817940108198, Tokens per sec: 82928.60392677978, Loss: 2.266793966293335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6603 , TFLOPS: 102.13189086605941, Tokens per sec: 83454.59713542012, Loss: 2.2698354721069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6604 , TFLOPS: 100.49425240758794, Tokens per sec: 82116.44059443878, Loss: 2.277189254760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6605 , TFLOPS: 101.58274052051857, Tokens per sec: 83005.87225169118, Loss: 2.2888591289520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6606 , TFLOPS: 101.28125331714985, Tokens per sec: 82759.51929684755, Loss: 2.2951741218566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6607 , TFLOPS: 102.73769631335986, Tokens per sec: 83949.61636122888, Loss: 2.281015634536743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6608 , TFLOPS: 102.11815866324875, Tokens per sec: 83443.37619900516, Loss: 2.269360065460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6609 , TFLOPS: 102.58910450726079, Tokens per sec: 83828.19817137194, Loss: 2.2617740631103516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6610 , TFLOPS: 102.13142748102163, Tokens per sec: 83454.21849157714, Loss: 2.286893844604492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6611 , TFLOPS: 102.09279026368294, Tokens per sec: 83422.64702667925, Loss: 2.261868953704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6612 , TFLOPS: 102.66296559609621, Tokens per sec: 83888.55196841294, Loss: 2.2743074893951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6613 , TFLOPS: 102.06601225413333, Tokens per sec: 83400.76602574896, Loss: 2.2969202995300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6614 , TFLOPS: 101.70337420766501, Tokens per sec: 83104.4451428459, Loss: 2.26257586479187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6615 , TFLOPS: 102.1138103742902, Tokens per sec: 83439.82310016222, Loss: 2.268481969833374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6616 , TFLOPS: 102.00095609129801, Tokens per sec: 83347.60696040161, Loss: 2.2679452896118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6617 , TFLOPS: 102.74035393502757, Tokens per sec: 83951.78797230683, Loss: 2.267871379852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6618 , TFLOPS: 102.08151058393264, Tokens per sec: 83413.43010999048, Loss: 2.269416332244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6619 , TFLOPS: 101.99223272604729, Tokens per sec: 83340.47887409585, Loss: 2.2695019245147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6620 , TFLOPS: 102.37506627556931, Tokens per sec: 83653.30202242259, Loss: 2.264488935470581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6621 , TFLOPS: 102.70036188155025, Tokens per sec: 83919.10943592347, Loss: 2.276704788208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6622 , TFLOPS: 102.89154682458867, Tokens per sec: 84075.33157441842, Loss: 2.273174524307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6623 , TFLOPS: 102.77134653738244, Tokens per sec: 83977.11282551155, Loss: 2.2668545246124268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6624 , TFLOPS: 101.69722923222537, Tokens per sec: 83099.42392522843, Loss: 2.271644115447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6625 , TFLOPS: 103.26587593748107, Tokens per sec: 84381.20552865142, Loss: 2.255601167678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6626 , TFLOPS: 103.40432216240568, Tokens per sec: 84494.3335029602, Loss: 2.2775075435638428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6627 , TFLOPS: 100.21106051666104, Tokens per sec: 81885.03721035461, Loss: 2.272101879119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6628 , TFLOPS: 103.3508792921083, Tokens per sec: 84450.66395789834, Loss: 2.2753102779388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6629 , TFLOPS: 102.77444134078496, Tokens per sec: 83979.64166903908, Loss: 2.26607346534729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6630 , TFLOPS: 102.08824155901706, Tokens per sec: 83418.93016300269, Loss: 2.281348466873169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6631 , TFLOPS: 101.37077529612118, Tokens per sec: 82832.67001036578, Loss: 2.2479450702667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6632 , TFLOPS: 103.287326652249, Tokens per sec: 84398.73345987883, Loss: 2.28857421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6633 , TFLOPS: 100.43638481747506, Tokens per sec: 82069.15549691327, Loss: 2.285569667816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6634 , TFLOPS: 102.78506575701668, Tokens per sec: 83988.32315303881, Loss: 2.2994444370269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6635 , TFLOPS: 100.93965925737639, Tokens per sec: 82480.39399719329, Loss: 2.303738832473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6636 , TFLOPS: 103.30291640973158, Tokens per sec: 84411.4722520339, Loss: 2.275599241256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6637 , TFLOPS: 100.01286802725028, Tokens per sec: 81723.08902532852, Loss: 2.2620160579681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6638 , TFLOPS: 102.09324406048411, Tokens per sec: 83423.01783572725, Loss: 2.272416353225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6639 , TFLOPS: 102.07260044664824, Tokens per sec: 83406.14940744806, Loss: 2.282829761505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6640 , TFLOPS: 100.8496185777946, Tokens per sec: 82406.81944005369, Loss: 2.3000967502593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6641 , TFLOPS: 102.69365293089125, Tokens per sec: 83913.62737963625, Loss: 2.2644503116607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6642 , TFLOPS: 101.04270005306121, Tokens per sec: 82564.5912838534, Loss: 2.273959159851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6643 , TFLOPS: 102.21993855983898, Tokens per sec: 83526.54317255666, Loss: 2.267946243286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6644 , TFLOPS: 102.72356750428217, Tokens per sec: 83938.0713476236, Loss: 2.29695987701416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6645 , TFLOPS: 102.0153975922823, Tokens per sec: 83359.40748261318, Loss: 2.2862765789031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6646 , TFLOPS: 102.8032979470495, Tokens per sec: 84003.22114485296, Loss: 2.27309513092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6647 , TFLOPS: 103.41488831518231, Tokens per sec: 84502.96737839095, Loss: 2.2802693843841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6648 , TFLOPS: 100.73137545432243, Tokens per sec: 82310.1999400158, Loss: 2.274482488632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6649 , TFLOPS: 102.88226303901371, Tokens per sec: 84067.74555423921, Loss: 2.2763943672180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6650 , TFLOPS: 102.12461236327813, Tokens per sec: 83448.64968343219, Loss: 2.2712221145629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6651 , TFLOPS: 102.89930758879034, Tokens per sec: 84081.67309462745, Loss: 2.2786998748779297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6652 , TFLOPS: 101.29405439645825, Tokens per sec: 82769.97938827953, Loss: 2.2828946113586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6653 , TFLOPS: 102.65805871370328, Tokens per sec: 83884.54242849529, Loss: 2.287832021713257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6654 , TFLOPS: 102.16083371769122, Tokens per sec: 83478.24708454373, Loss: 2.2801363468170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6655 , TFLOPS: 102.63318106202544, Tokens per sec: 83864.2142589016, Loss: 2.303677797317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6656 , TFLOPS: 102.03755693272349, Tokens per sec: 83377.51445011984, Loss: 2.26625919342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6657 , TFLOPS: 101.87976853299236, Tokens per sec: 83248.58148686479, Loss: 2.2911715507507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6658 , TFLOPS: 101.49176886963299, Tokens per sec: 82931.53697393397, Loss: 2.2932987213134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6659 , TFLOPS: 102.59988837042555, Tokens per sec: 83837.0099435652, Loss: 2.269932270050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6660 , TFLOPS: 102.14684058643883, Tokens per sec: 83466.81293678217, Loss: 2.292142868041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6661 , TFLOPS: 102.72231449545936, Tokens per sec: 83937.04748185916, Loss: 2.2665677070617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6662 , TFLOPS: 101.57320579626855, Tokens per sec: 82998.08118306086, Loss: 2.2767181396484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6663 , TFLOPS: 103.34098866326897, Tokens per sec: 84442.58207046629, Loss: 2.271275043487549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6664 , TFLOPS: 103.36232091697327, Tokens per sec: 84460.01320410914, Loss: 2.2612404823303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6665 , TFLOPS: 101.01481384046859, Tokens per sec: 82541.80474168899, Loss: 2.2862613201141357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6666 , TFLOPS: 103.41133173954393, Tokens per sec: 84500.06120888243, Loss: 2.275437116622925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6667 , TFLOPS: 102.53532477520653, Tokens per sec: 83784.25336789695, Loss: 2.266611337661743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6668 , TFLOPS: 102.10469028827951, Tokens per sec: 83432.370843111, Loss: 2.2593822479248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6669 , TFLOPS: 100.71656582941556, Tokens per sec: 82298.09861427073, Loss: 2.2861692905426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6670 , TFLOPS: 103.39339237920959, Tokens per sec: 84485.40249575303, Loss: 2.273815631866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6671 , TFLOPS: 101.52367833631162, Tokens per sec: 82957.6110205798, Loss: 2.276815891265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6672 , TFLOPS: 102.7846684023493, Tokens per sec: 83987.99846431128, Loss: 2.274744749069214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6673 , TFLOPS: 101.48021630083736, Tokens per sec: 82922.09707258145, Loss: 2.282649278640747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6674 , TFLOPS: 103.32449304456223, Tokens per sec: 84429.10307578601, Loss: 2.275981903076172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6675 , TFLOPS: 99.51571416508017, Tokens per sec: 81316.85180667038, Loss: 2.2772722244262695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6676 , TFLOPS: 102.74677364830501, Tokens per sec: 83957.03368528395, Loss: 2.2978157997131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6677 , TFLOPS: 102.05138071458413, Tokens per sec: 83388.81021813398, Loss: 2.2990782260894775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6678 , TFLOPS: 100.95582531004267, Tokens per sec: 82493.60369497811, Loss: 2.2905900478363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6679 , TFLOPS: 102.76736928142563, Tokens per sec: 83973.86290728555, Loss: 2.2782068252563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6680 , TFLOPS: 100.92541140409153, Tokens per sec: 82468.75170950186, Loss: 2.2809078693389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6681 , TFLOPS: 101.39575429838101, Tokens per sec: 82853.080996129, Loss: 2.2926859855651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6682 , TFLOPS: 102.78356709616556, Tokens per sec: 83987.09855867841, Loss: 2.2562010288238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6683 , TFLOPS: 102.07120728507984, Tokens per sec: 83405.01101926735, Loss: 2.287010431289673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6684 , TFLOPS: 102.65866766070569, Tokens per sec: 83885.04001476664, Loss: 2.2796475887298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6685 , TFLOPS: 103.41930165701251, Tokens per sec: 84506.57363360994, Loss: 2.29386568069458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6686 , TFLOPS: 101.51518631069939, Tokens per sec: 82950.67196784787, Loss: 2.275139808654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6687 , TFLOPS: 102.26775266385087, Tokens per sec: 83565.61330778932, Loss: 2.2625513076782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6688 , TFLOPS: 102.73983045407141, Tokens per sec: 83951.36022254195, Loss: 2.278388500213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6689 , TFLOPS: 102.02130575068523, Tokens per sec: 83364.23519093374, Loss: 2.2558932304382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6690 , TFLOPS: 101.87712178803196, Tokens per sec: 83246.41876342437, Loss: 2.289224863052368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6691 , TFLOPS: 102.83753573906591, Tokens per sec: 84031.19772606876, Loss: 2.2906699180603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6692 , TFLOPS: 102.02406571330455, Tokens per sec: 83366.49042743775, Loss: 2.282630681991577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6693 , TFLOPS: 102.73937538092456, Tokens per sec: 83950.98837055903, Loss: 2.2730839252471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6694 , TFLOPS: 101.92385207840094, Tokens per sec: 83284.60328662954, Loss: 2.2891604900360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6695 , TFLOPS: 101.90906843669929, Tokens per sec: 83272.52319243034, Loss: 2.275899887084961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6696 , TFLOPS: 101.50529126020305, Tokens per sec: 82942.58646736604, Loss: 2.268907070159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6697 , TFLOPS: 102.6202012568222, Tokens per sec: 83853.60812594013, Loss: 2.2780673503875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6698 , TFLOPS: 102.19053555350997, Tokens per sec: 83502.51721918452, Loss: 2.283094882965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6699 , TFLOPS: 102.19538350671866, Tokens per sec: 83506.47860654876, Loss: 2.247109889984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6700 , TFLOPS: 101.58304877667148, Tokens per sec: 83006.12413572894, Loss: 2.27194881439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6701 , TFLOPS: 102.76558680386505, Tokens per sec: 83972.40639898575, Loss: 2.2884321212768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6702 , TFLOPS: 103.34602480795826, Tokens per sec: 84446.69723393381, Loss: 2.2845029830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6703 , TFLOPS: 100.89280582957848, Tokens per sec: 82442.10885522507, Loss: 2.2695789337158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6704 , TFLOPS: 103.41271271666295, Tokens per sec: 84501.1896408359, Loss: 2.2884411811828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6705 , TFLOPS: 102.82950970543301, Tokens per sec: 84024.63944737882, Loss: 2.2919507026672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6706 , TFLOPS: 101.98373759593657, Tokens per sec: 83333.53728459828, Loss: 2.2874348163604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6707 , TFLOPS: 101.43067804211182, Tokens per sec: 82881.61808614859, Loss: 2.2572133541107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6708 , TFLOPS: 103.369171240216, Tokens per sec: 84465.61077957378, Loss: 2.281736373901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6709 , TFLOPS: 101.36385252718854, Tokens per sec: 82827.01323766299, Loss: 2.2942416667938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6710 , TFLOPS: 101.541410436576, Tokens per sec: 82972.1003761707, Loss: 2.287360191345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6711 , TFLOPS: 101.60290118237, Tokens per sec: 83022.3460474716, Loss: 2.280897378921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6712 , TFLOPS: 103.3556892635508, Tokens per sec: 84454.59430938368, Loss: 2.274157762527466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6713 , TFLOPS: 100.07647807862428, Tokens per sec: 81775.06643577461, Loss: 2.29296612739563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6714 , TFLOPS: 102.85911571575122, Tokens per sec: 84048.83128053632, Loss: 2.2772085666656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6715 , TFLOPS: 101.96817015808467, Tokens per sec: 83320.81673038799, Loss: 2.2829251289367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6716 , TFLOPS: 100.17856222262795, Tokens per sec: 81858.48201771967, Loss: 2.2708773612976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6717 , TFLOPS: 102.64847616299299, Tokens per sec: 83876.71227963311, Loss: 2.282167911529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6718 , TFLOPS: 101.636263991535, Tokens per sec: 83049.60765767543, Loss: 2.2944202423095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6719 , TFLOPS: 102.29065841395003, Tokens per sec: 83584.3301858418, Loss: 2.269261360168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6720 , TFLOPS: 102.74524102767322, Tokens per sec: 83955.78134151253, Loss: 2.2741482257843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6721 , TFLOPS: 102.05412697689557, Tokens per sec: 83391.05425976374, Loss: 2.2676031589508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6722 , TFLOPS: 102.7591533223946, Tokens per sec: 83967.14943566371, Loss: 2.2461252212524414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6723 , TFLOPS: 103.38205089740845, Tokens per sec: 84476.13507901761, Loss: 2.2707760334014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6724 , TFLOPS: 101.36208129666862, Tokens per sec: 82825.56591960964, Loss: 2.27091383934021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6725 , TFLOPS: 102.24568660750467, Tokens per sec: 83547.58256511802, Loss: 2.28808856010437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6726 , TFLOPS: 102.71578044725881, Tokens per sec: 83931.70834287311, Loss: 2.2748374938964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6727 , TFLOPS: 102.7774725355624, Tokens per sec: 83982.11853632162, Loss: 2.2763280868530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6728 , TFLOPS: 99.16021649288264, Tokens per sec: 81026.36550739357, Loss: 2.285364866256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6729 , TFLOPS: 102.76838712455658, Tokens per sec: 83974.69461310947, Loss: 2.282547950744629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6730 , TFLOPS: 102.64451860846029, Tokens per sec: 83873.47845994792, Loss: 2.248128890991211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6731 , TFLOPS: 101.84304054763048, Tokens per sec: 83218.57010456313, Loss: 2.274531602859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6732 , TFLOPS: 102.66914950319479, Tokens per sec: 83893.60499809112, Loss: 2.2616608142852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6733 , TFLOPS: 101.37355986940189, Tokens per sec: 82834.94535687474, Loss: 2.257323741912842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6734 , TFLOPS: 102.25621292543575, Tokens per sec: 83556.18389047103, Loss: 2.281344175338745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6735 , TFLOPS: 101.83157572129528, Tokens per sec: 83209.20189983376, Loss: 2.2824645042419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6736 , TFLOPS: 102.83614401878309, Tokens per sec: 84030.06051559957, Loss: 2.2860970497131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6737 , TFLOPS: 102.17956132324234, Tokens per sec: 83493.54989313103, Loss: 2.2970046997070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6738 , TFLOPS: 101.62758139350358, Tokens per sec: 83042.512883314, Loss: 2.2999978065490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6739 , TFLOPS: 102.75320143709662, Tokens per sec: 83962.28599697156, Loss: 2.2706260681152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6740 , TFLOPS: 103.3678702968559, Tokens per sec: 84464.54774526498, Loss: 2.278637170791626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6741 , TFLOPS: 100.21992154789967, Tokens per sec: 81892.27778708294, Loss: 2.276005268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6742 , TFLOPS: 103.31313665841182, Tokens per sec: 84419.82348032312, Loss: 2.272059202194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6743 , TFLOPS: 102.12639768321897, Tokens per sec: 83450.10851431408, Loss: 2.2839455604553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6744 , TFLOPS: 102.01823977730034, Tokens per sec: 83361.72990515576, Loss: 2.2814788818359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6745 , TFLOPS: 101.4250742543734, Tokens per sec: 82877.0390869332, Loss: 2.2697274684906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6746 , TFLOPS: 103.44539641749685, Tokens per sec: 84527.89633414059, Loss: 2.273909568786621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6747 , TFLOPS: 101.57438287612037, Tokens per sec: 82999.04300530851, Loss: 2.292253017425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6748 , TFLOPS: 102.77959020381671, Tokens per sec: 83983.84893756584, Loss: 2.263028383255005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6749 , TFLOPS: 100.91241437001499, Tokens per sec: 82458.13149838448, Loss: 2.272815227508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6750 , TFLOPS: 103.43054335063285, Tokens per sec: 84515.75951085406, Loss: 2.27582049369812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6751 , TFLOPS: 100.15459846586641, Tokens per sec: 81838.90061518784, Loss: 2.2898030281066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6752 , TFLOPS: 102.88117416304807, Tokens per sec: 84066.85580565754, Loss: 2.2709341049194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6753 , TFLOPS: 101.53063013850227, Tokens per sec: 82963.2915171051, Loss: 2.2711620330810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6754 , TFLOPS: 101.05030445221792, Tokens per sec: 82570.80503415898, Loss: 2.2521045207977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6755 , TFLOPS: 103.35399519999218, Tokens per sec: 84453.21004644087, Loss: 2.275379180908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6756 , TFLOPS: 100.43130093172881, Tokens per sec: 82065.00132299915, Loss: 2.263181209564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6757 , TFLOPS: 102.29257510599807, Tokens per sec: 83585.89636425431, Loss: 2.2577426433563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6758 , TFLOPS: 102.0783787810237, Tokens per sec: 83410.87103321381, Loss: 2.2884044647216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6759 , TFLOPS: 103.41674991340099, Tokens per sec: 84504.48853821724, Loss: 2.2940590381622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6760 , TFLOPS: 101.4291834203738, Tokens per sec: 82880.39679225149, Loss: 2.2849555015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6761 , TFLOPS: 103.50257042220112, Tokens per sec: 84574.61468517427, Loss: 2.2728543281555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6762 , TFLOPS: 101.48398147751313, Tokens per sec: 82925.17369536751, Loss: 2.2688136100769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6763 , TFLOPS: 102.105214283769, Tokens per sec: 83432.7990133143, Loss: 2.2657370567321777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6764 , TFLOPS: 102.87979731917211, Tokens per sec: 84065.73075108338, Loss: 2.264975070953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6765 , TFLOPS: 102.79657437011942, Tokens per sec: 83997.72713706312, Loss: 2.2627639770507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6766 , TFLOPS: 100.85768597641747, Tokens per sec: 82413.41151914177, Loss: 2.2941513061523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6767 , TFLOPS: 102.82151724763554, Tokens per sec: 84018.10860436835, Loss: 2.293079137802124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6768 , TFLOPS: 101.9986205746675, Tokens per sec: 83345.69854964127, Loss: 2.293226480484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6769 , TFLOPS: 102.77082631254135, Tokens per sec: 83976.68773640212, Loss: 2.2687840461730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6770 , TFLOPS: 102.686839150136, Tokens per sec: 83908.0596639788, Loss: 2.306187868118286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6771 , TFLOPS: 101.92846676890031, Tokens per sec: 83288.37406902928, Loss: 2.2959840297698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6772 , TFLOPS: 102.05366510824724, Tokens per sec: 83390.67685500163, Loss: 2.303347587585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6773 , TFLOPS: 102.20438819202647, Tokens per sec: 83513.83656671499, Loss: 2.290379762649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6774 , TFLOPS: 102.71551212262835, Tokens per sec: 83931.48908791029, Loss: 2.2780561447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6775 , TFLOPS: 103.179234454768, Tokens per sec: 84310.40854277638, Loss: 2.260049343109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6776 , TFLOPS: 100.94804730412103, Tokens per sec: 82487.24808611585, Loss: 2.2872562408447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6777 , TFLOPS: 101.3454727517591, Tokens per sec: 82811.9946500221, Loss: 2.2754852771759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6778 , TFLOPS: 103.4977876321034, Tokens per sec: 84570.70654426544, Loss: 2.2782251834869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6779 , TFLOPS: 100.79693847702316, Tokens per sec: 82363.77317360691, Loss: 2.263032913208008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6780 , TFLOPS: 103.45298863176268, Tokens per sec: 84534.10012785818, Loss: 2.2699389457702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6781 , TFLOPS: 102.33660799944833, Tokens per sec: 83621.87677500003, Loss: 2.2769885063171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6782 , TFLOPS: 102.0714433465795, Tokens per sec: 83405.2039111955, Loss: 2.28120756149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6783 , TFLOPS: 101.50911848743543, Tokens per sec: 82945.71379325922, Loss: 2.2610881328582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6784 , TFLOPS: 102.701385998217, Tokens per sec: 83919.94626801497, Loss: 2.270681858062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6785 , TFLOPS: 101.57846175551907, Tokens per sec: 83002.3759626649, Loss: 2.280790090560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6786 , TFLOPS: 102.10056226084208, Tokens per sec: 83428.99772562699, Loss: 2.262979745864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6787 , TFLOPS: 100.33815957591932, Tokens per sec: 81988.8931234954, Loss: 2.288614511489868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6788 , TFLOPS: 103.39033019813901, Tokens per sec: 84482.90030877272, Loss: 2.283155918121338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6789 , TFLOPS: 100.81149646887918, Tokens per sec: 82375.66888350854, Loss: 2.2754135131835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6790 , TFLOPS: 102.08787094166566, Tokens per sec: 83418.62732202432, Loss: 2.279247283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6791 , TFLOPS: 101.40008216349518, Tokens per sec: 82856.61740612295, Loss: 2.2904670238494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6792 , TFLOPS: 100.55818288789821, Tokens per sec: 82168.67983561728, Loss: 2.2676963806152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6793 , TFLOPS: 102.59785236580187, Tokens per sec: 83835.34627177578, Loss: 2.283991575241089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6794 , TFLOPS: 100.44749750987947, Tokens per sec: 82078.23596394307, Loss: 2.2759766578674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6795 , TFLOPS: 102.26180038782859, Tokens per sec: 83560.74954982629, Loss: 2.2817776203155518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6796 , TFLOPS: 101.35985263051056, Tokens per sec: 82823.74481912091, Loss: 2.2749040126800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6797 , TFLOPS: 102.50124760274733, Tokens per sec: 83756.40803305607, Loss: 2.2916946411132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6798 , TFLOPS: 101.62092044068964, Tokens per sec: 83037.07004730155, Loss: 2.2783284187316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6799 , TFLOPS: 102.90285816793082, Tokens per sec: 84084.57436424318, Loss: 2.2869348526000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6800 , TFLOPS: 101.61312702383385, Tokens per sec: 83030.70184576829, Loss: 2.2869365215301514 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/model.pt +[rank0]:[2024-08-28 04:15:28,372] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007747250987449661, 'preprocessing_with_comm': 0.001740255014738068, 'state_converting': 3.3370461059967056, : 3.348184228001628}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800, took 15.77s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0004800 + eval ppl=8.307998657226562, eval loss=2.1172187328338623 +------------------------------------------------------------------ +iteration: 6801 , TFLOPS: 100.7291530457468, Tokens per sec: 82308.38395275877, Loss: 2.274066686630249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6802 , TFLOPS: 102.18946940658034, Tokens per sec: 83501.64604308325, Loss: 2.2933218479156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6803 , TFLOPS: 101.94813231779209, Tokens per sec: 83304.44329526502, Loss: 2.275512933731079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6804 , TFLOPS: 100.13000417845878, Tokens per sec: 81818.80398983377, Loss: 2.269052743911743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6805 , TFLOPS: 102.22110831609855, Tokens per sec: 83527.49901050857, Loss: 2.2849225997924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6806 , TFLOPS: 100.33749829716851, Tokens per sec: 81988.3527756053, Loss: 2.2846124172210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6807 , TFLOPS: 101.86947329662664, Tokens per sec: 83240.16898420738, Loss: 2.2713942527770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6808 , TFLOPS: 102.55635071497481, Tokens per sec: 83801.43420453745, Loss: 2.2911744117736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6809 , TFLOPS: 101.58263729971421, Tokens per sec: 83005.7879073148, Loss: 2.281085252761841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6810 , TFLOPS: 103.45616529470466, Tokens per sec: 84536.69586092266, Loss: 2.257761001586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6811 , TFLOPS: 102.18049629971468, Tokens per sec: 83494.31388647499, Loss: 2.2717955112457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6812 , TFLOPS: 102.86508740361613, Tokens per sec: 84053.71090041568, Loss: 2.27815318107605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6813 , TFLOPS: 102.55248630284404, Tokens per sec: 83798.27649390658, Loss: 2.2868905067443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6814 , TFLOPS: 102.4922365340932, Tokens per sec: 83749.04485689323, Loss: 2.2978272438049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6815 , TFLOPS: 102.00842834794649, Tokens per sec: 83353.71273366209, Loss: 2.2600595951080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6816 , TFLOPS: 102.42380684611045, Tokens per sec: 83693.12919730567, Loss: 2.2721946239471436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6817 , TFLOPS: 102.12872727137172, Tokens per sec: 83452.01208076262, Loss: 2.2630183696746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6818 , TFLOPS: 102.79011226544256, Tokens per sec: 83992.44678498228, Loss: 2.2917885780334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6819 , TFLOPS: 102.67828553866732, Tokens per sec: 83901.07028785821, Loss: 2.264971971511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6820 , TFLOPS: 102.74940777280078, Tokens per sec: 83959.18609621783, Loss: 2.26702880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6821 , TFLOPS: 102.06702165640037, Tokens per sec: 83401.59083432569, Loss: 2.2868459224700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6822 , TFLOPS: 102.57862010169163, Tokens per sec: 83819.63109368877, Loss: 2.2988884449005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6823 , TFLOPS: 101.98787389311917, Tokens per sec: 83336.91715950395, Loss: 2.264233112335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6824 , TFLOPS: 103.42440251734561, Tokens per sec: 84510.74167789595, Loss: 2.2818515300750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6825 , TFLOPS: 102.66729123375467, Tokens per sec: 83892.08655829547, Loss: 2.274643898010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6826 , TFLOPS: 101.28440743872363, Tokens per sec: 82762.09661077979, Loss: 2.2957639694213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6827 , TFLOPS: 102.83709452730146, Tokens per sec: 84030.83720057821, Loss: 2.2901692390441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6828 , TFLOPS: 102.09817751440258, Tokens per sec: 83427.04908792244, Loss: 2.2583088874816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6829 , TFLOPS: 102.02514771789566, Tokens per sec: 83367.37456124242, Loss: 2.284341335296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6830 , TFLOPS: 102.69875564890171, Tokens per sec: 83917.79694187824, Loss: 2.275287628173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6831 , TFLOPS: 101.57349228226015, Tokens per sec: 82998.3152781394, Loss: 2.2970755100250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6832 , TFLOPS: 102.77055624622129, Tokens per sec: 83976.46705825986, Loss: 2.296503782272339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6833 , TFLOPS: 101.88027887711486, Tokens per sec: 83248.9985021847, Loss: 2.2856602668762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6834 , TFLOPS: 100.69804598134142, Tokens per sec: 82282.96556966606, Loss: 2.2850406169891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6835 , TFLOPS: 102.00997856078816, Tokens per sec: 83354.97945248104, Loss: 2.2708358764648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6836 , TFLOPS: 102.22112000414627, Tokens per sec: 83527.50856111324, Loss: 2.2737770080566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6837 , TFLOPS: 101.34202589425648, Tokens per sec: 82809.17813402669, Loss: 2.284270763397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6838 , TFLOPS: 101.59868363098884, Tokens per sec: 83018.89977766863, Loss: 2.286972999572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6839 , TFLOPS: 101.40302559920875, Tokens per sec: 82859.02256321529, Loss: 2.278383731842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6840 , TFLOPS: 102.1045026534083, Tokens per sec: 83432.21752182757, Loss: 2.2727179527282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6841 , TFLOPS: 101.97670457359683, Tokens per sec: 83327.79042099837, Loss: 2.2883048057556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6842 , TFLOPS: 101.01490923757474, Tokens per sec: 82541.88269312018, Loss: 2.2738935947418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6843 , TFLOPS: 102.30390468067668, Tokens per sec: 83595.15405137339, Loss: 2.27809739112854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6844 , TFLOPS: 100.8009529464838, Tokens per sec: 82367.05349994484, Loss: 2.2785539627075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6845 , TFLOPS: 101.75016575099798, Tokens per sec: 83142.67971742476, Loss: 2.262874126434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6846 , TFLOPS: 102.48364425253925, Tokens per sec: 83742.02388245049, Loss: 2.2735483646392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6847 , TFLOPS: 101.29495912117429, Tokens per sec: 82770.7186621347, Loss: 2.2723703384399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6848 , TFLOPS: 103.32849940407712, Tokens per sec: 84432.37677527849, Loss: 2.2852113246917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6849 , TFLOPS: 102.24649767126493, Tokens per sec: 83548.24530619517, Loss: 2.274986743927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6850 , TFLOPS: 102.0361011617191, Tokens per sec: 83376.32490215728, Loss: 2.2973525524139404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6851 , TFLOPS: 102.79517659138556, Tokens per sec: 83996.58497607766, Loss: 2.276352643966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6852 , TFLOPS: 103.3702359247962, Tokens per sec: 84466.48076075169, Loss: 2.288133382797241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6853 , TFLOPS: 102.0548233036773, Tokens per sec: 83391.62324630213, Loss: 2.2716925144195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6854 , TFLOPS: 102.56466571413105, Tokens per sec: 83808.22860439513, Loss: 2.2714176177978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6855 , TFLOPS: 102.13548477122683, Tokens per sec: 83457.53380784775, Loss: 2.3014745712280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6856 , TFLOPS: 102.00154065303163, Tokens per sec: 83348.08462084211, Loss: 2.2772231101989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6857 , TFLOPS: 103.41162393930234, Tokens per sec: 84500.29997282694, Loss: 2.2820067405700684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6858 , TFLOPS: 102.0621576718695, Tokens per sec: 83397.61634735542, Loss: 2.277735948562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6859 , TFLOPS: 102.8094712814661, Tokens per sec: 84008.26553532053, Loss: 2.26286244392395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6860 , TFLOPS: 102.5053051237289, Tokens per sec: 83759.72353789999, Loss: 2.287212610244751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6861 , TFLOPS: 101.48008705600324, Tokens per sec: 82921.99146330045, Loss: 2.28387713432312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6862 , TFLOPS: 103.22761784337571, Tokens per sec: 84349.94385510698, Loss: 2.2764289379119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6863 , TFLOPS: 102.18841808409968, Tokens per sec: 83500.78698042085, Loss: 2.279934883117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6864 , TFLOPS: 101.8776538642485, Tokens per sec: 83246.85353659776, Loss: 2.26515531539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6865 , TFLOPS: 102.09920689346991, Tokens per sec: 83427.89022005691, Loss: 2.2703909873962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6866 , TFLOPS: 102.70954033811275, Tokens per sec: 83926.60938905517, Loss: 2.2639498710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6867 , TFLOPS: 102.85136184972544, Tokens per sec: 84042.4953970044, Loss: 2.254814624786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6868 , TFLOPS: 101.88634419855195, Tokens per sec: 83253.95463246621, Loss: 2.289842367172241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6869 , TFLOPS: 102.16792614907797, Tokens per sec: 83484.04249280528, Loss: 2.2608413696289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6870 , TFLOPS: 101.25162396225836, Tokens per sec: 82735.30838823826, Loss: 2.2614493370056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6871 , TFLOPS: 101.92412857607206, Tokens per sec: 83284.82921999431, Loss: 2.276895523071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6872 , TFLOPS: 102.07790663386352, Tokens per sec: 83410.48522961508, Loss: 2.296858310699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6873 , TFLOPS: 101.22902555201458, Tokens per sec: 82716.84264549323, Loss: 2.2836616039276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6874 , TFLOPS: 101.60341520518345, Tokens per sec: 83022.76606874459, Loss: 2.2809910774230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6875 , TFLOPS: 101.94782006202975, Tokens per sec: 83304.18814304343, Loss: 2.2842538356781006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6876 , TFLOPS: 100.98655161034489, Tokens per sec: 82518.7109459204, Loss: 2.2809784412384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6877 , TFLOPS: 101.26973812665675, Tokens per sec: 82750.10993826827, Loss: 2.2658283710479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6878 , TFLOPS: 102.21976248188781, Tokens per sec: 83526.39929473012, Loss: 2.294029951095581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6879 , TFLOPS: 100.76970982385383, Tokens per sec: 82341.52394017436, Loss: 2.2794179916381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6880 , TFLOPS: 101.5791458804074, Tokens per sec: 83002.93497872207, Loss: 2.272181510925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6881 , TFLOPS: 101.34297891256384, Tokens per sec: 82809.9568698185, Loss: 2.2739500999450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6882 , TFLOPS: 100.90083804265814, Tokens per sec: 82448.67218330002, Loss: 2.255317449569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6883 , TFLOPS: 101.78176115879367, Tokens per sec: 83168.49713847277, Loss: 2.2751967906951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6884 , TFLOPS: 102.57916776165955, Tokens per sec: 83820.07860074658, Loss: 2.2810728549957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6885 , TFLOPS: 102.32596901461878, Tokens per sec: 83613.18338662396, Loss: 2.276346206665039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6886 , TFLOPS: 102.80020873078487, Tokens per sec: 84000.69686671965, Loss: 2.286280870437622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6887 , TFLOPS: 103.34752667850937, Tokens per sec: 84447.92445102271, Loss: 2.2955782413482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6888 , TFLOPS: 102.8858495920612, Tokens per sec: 84070.67621906042, Loss: 2.2775864601135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6889 , TFLOPS: 102.87056712106708, Tokens per sec: 84058.18851860531, Loss: 2.30145525932312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6890 , TFLOPS: 103.35935693321314, Tokens per sec: 84457.591256679, Loss: 2.264126777648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6891 , TFLOPS: 101.29198920676994, Tokens per sec: 82768.29186862252, Loss: 2.257396697998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6892 , TFLOPS: 102.57915768891314, Tokens per sec: 83820.07037004619, Loss: 2.2557685375213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6893 , TFLOPS: 102.75812823074835, Tokens per sec: 83966.31180689136, Loss: 2.2800610065460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6894 , TFLOPS: 103.2535632688104, Tokens per sec: 84371.14453012432, Loss: 2.2835748195648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6895 , TFLOPS: 102.84010869008976, Tokens per sec: 84033.300150584, Loss: 2.2796361446380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6896 , TFLOPS: 102.5212342022929, Tokens per sec: 83772.7395980456, Loss: 2.2658281326293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6897 , TFLOPS: 101.89726888544695, Tokens per sec: 83262.88147535457, Loss: 2.2829601764678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6898 , TFLOPS: 102.67447305848451, Tokens per sec: 83897.95501215885, Loss: 2.273223400115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6899 , TFLOPS: 102.0723976288993, Tokens per sec: 83405.9836798444, Loss: 2.281136989593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6900 , TFLOPS: 103.37295712689584, Tokens per sec: 84468.70432503727, Loss: 2.281276226043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6901 , TFLOPS: 102.17951995421484, Tokens per sec: 83493.51608943372, Loss: 2.273726224899292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6902 , TFLOPS: 101.31582422830692, Tokens per sec: 82787.76807828843, Loss: 2.2535221576690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6903 , TFLOPS: 102.80252104606029, Tokens per sec: 84002.58631905547, Loss: 2.2837626934051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6904 , TFLOPS: 102.2757119063417, Tokens per sec: 83572.11700971771, Loss: 2.2882676124572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6905 , TFLOPS: 102.91435800334618, Tokens per sec: 84093.97117579317, Loss: 2.2882657051086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6906 , TFLOPS: 101.43219120067114, Tokens per sec: 82882.85452695958, Loss: 2.2788681983947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6907 , TFLOPS: 102.07261339855677, Tokens per sec: 83406.15999078593, Loss: 2.2684245109558105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6908 , TFLOPS: 101.98597320198382, Tokens per sec: 83335.3640558491, Loss: 2.3095972537994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6909 , TFLOPS: 101.11389127805666, Tokens per sec: 82622.76346642234, Loss: 2.2881386280059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6910 , TFLOPS: 101.52595227458254, Tokens per sec: 82959.46911407735, Loss: 2.264613628387451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6911 , TFLOPS: 102.56003994141722, Tokens per sec: 83804.44876643267, Loss: 2.2593350410461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6912 , TFLOPS: 100.32590483410353, Tokens per sec: 81978.87945849259, Loss: 2.259909152984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6913 , TFLOPS: 101.5768436682073, Tokens per sec: 83001.05378186917, Loss: 2.2803592681884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6914 , TFLOPS: 101.45595110497408, Tokens per sec: 82902.26935639986, Loss: 2.302957534790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6915 , TFLOPS: 100.82418283672588, Tokens per sec: 82386.03524125222, Loss: 2.2701926231384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6916 , TFLOPS: 101.9616658267543, Tokens per sec: 83315.50187382156, Loss: 2.2714390754699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6917 , TFLOPS: 101.89606269909163, Tokens per sec: 83261.89586943365, Loss: 2.28281831741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6918 , TFLOPS: 101.31061475578736, Tokens per sec: 82783.51128418893, Loss: 2.2734203338623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6919 , TFLOPS: 102.14669100709193, Tokens per sec: 83466.6907116473, Loss: 2.2734711170196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6920 , TFLOPS: 100.48842839394281, Tokens per sec: 82111.68164296591, Loss: 2.2656338214874268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6921 , TFLOPS: 102.40578445865233, Tokens per sec: 83678.40264057611, Loss: 2.2553796768188477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6922 , TFLOPS: 102.22331975708903, Tokens per sec: 83529.30603586945, Loss: 2.2930641174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6923 , TFLOPS: 102.25438489012814, Tokens per sec: 83554.69015576337, Loss: 2.288538932800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6924 , TFLOPS: 102.6951120569816, Tokens per sec: 83914.81966912592, Loss: 2.284816265106201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6925 , TFLOPS: 103.49657585938206, Tokens per sec: 84569.71637358103, Loss: 2.303093194961548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6926 , TFLOPS: 103.34944058077028, Tokens per sec: 84449.48834982832, Loss: 2.2704505920410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6927 , TFLOPS: 103.42941301639502, Tokens per sec: 84514.83588565093, Loss: 2.275259494781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6928 , TFLOPS: 103.39907161745533, Tokens per sec: 84490.04314752022, Loss: 2.2786478996276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6929 , TFLOPS: 102.12657495322237, Tokens per sec: 83450.2533661972, Loss: 2.2822327613830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6930 , TFLOPS: 101.94098065708394, Tokens per sec: 83298.59948920026, Loss: 2.2687721252441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6931 , TFLOPS: 102.6755568346277, Tokens per sec: 83898.84059354433, Loss: 2.2854247093200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6932 , TFLOPS: 103.29407709609201, Tokens per sec: 84404.24942131483, Loss: 2.26938533782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6933 , TFLOPS: 101.89434795864562, Tokens per sec: 83260.49471087386, Loss: 2.2831199169158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6934 , TFLOPS: 102.61792981030548, Tokens per sec: 83851.75206851907, Loss: 2.2820911407470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6935 , TFLOPS: 102.76124319840481, Tokens per sec: 83968.85712714985, Loss: 2.2666099071502686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6936 , TFLOPS: 102.59446626274946, Tokens per sec: 83832.5793998056, Loss: 2.2730183601379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6937 , TFLOPS: 102.03727095501833, Tokens per sec: 83377.28077037523, Loss: 2.272914409637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6938 , TFLOPS: 103.33815331286374, Tokens per sec: 84440.26523266175, Loss: 2.258533477783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6939 , TFLOPS: 101.38884847001492, Tokens per sec: 82847.43806599925, Loss: 2.270458936691284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6940 , TFLOPS: 101.88551492376352, Tokens per sec: 83253.27701068915, Loss: 2.274043321609497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6941 , TFLOPS: 102.77771492776971, Tokens per sec: 83982.31660123394, Loss: 2.2819981575012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6942 , TFLOPS: 101.9844699439176, Tokens per sec: 83334.13570498589, Loss: 2.271289348602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6943 , TFLOPS: 102.822930650642, Tokens per sec: 84019.26353234913, Loss: 2.2815330028533936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6944 , TFLOPS: 101.41778646038381, Tokens per sec: 82871.0840428598, Loss: 2.282571315765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6945 , TFLOPS: 102.28567074416429, Tokens per sec: 83580.25463246585, Loss: 2.2706174850463867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6946 , TFLOPS: 101.99337690701212, Tokens per sec: 83341.41381381606, Loss: 2.2894036769866943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6947 , TFLOPS: 101.25809904523433, Tokens per sec: 82740.5993452214, Loss: 2.315307378768921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6948 , TFLOPS: 101.68673884016182, Tokens per sec: 83090.8519558266, Loss: 2.2727324962615967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6949 , TFLOPS: 102.71810873225922, Tokens per sec: 83933.61084448235, Loss: 2.2869760990142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6950 , TFLOPS: 101.70782744201038, Tokens per sec: 83108.08399526597, Loss: 2.2738230228424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6951 , TFLOPS: 100.86037491872436, Tokens per sec: 82415.60872312025, Loss: 2.2574143409729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6952 , TFLOPS: 102.17350659373005, Tokens per sec: 83488.60241778393, Loss: 2.285266876220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6953 , TFLOPS: 100.19547174755496, Tokens per sec: 81872.29922582742, Loss: 2.264518976211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6954 , TFLOPS: 102.08219801002848, Tokens per sec: 83413.99182354932, Loss: 2.2550148963928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6955 , TFLOPS: 101.98797099443325, Tokens per sec: 83336.99650348729, Loss: 2.298649787902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6956 , TFLOPS: 101.43929200923006, Tokens per sec: 82888.65678042413, Loss: 2.26936674118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6957 , TFLOPS: 102.12109946379019, Tokens per sec: 83445.77920283053, Loss: 2.2825512886047363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6958 , TFLOPS: 99.88319437769647, Tokens per sec: 81617.1293481814, Loss: 2.2640762329101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6959 , TFLOPS: 102.29606766276976, Tokens per sec: 83588.75022229887, Loss: 2.2620179653167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6960 , TFLOPS: 101.83309901134082, Tokens per sec: 83210.44661934295, Loss: 2.263296604156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6961 , TFLOPS: 102.12708430459581, Tokens per sec: 83450.66957031631, Loss: 2.2772653102874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6962 , TFLOPS: 102.72838145090073, Tokens per sec: 83942.00494732763, Loss: 2.2964394092559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6963 , TFLOPS: 102.85128110324344, Tokens per sec: 84042.42941697536, Loss: 2.2551486492156982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6964 , TFLOPS: 103.34137600430236, Tokens per sec: 84442.89857679572, Loss: 2.2759578227996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6965 , TFLOPS: 102.69781606437115, Tokens per sec: 83917.02918317125, Loss: 2.2799463272094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6966 , TFLOPS: 103.26570604064304, Tokens per sec: 84381.06670157153, Loss: 2.265904664993286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6967 , TFLOPS: 101.5168870039579, Tokens per sec: 82952.06164808913, Loss: 2.295581579208374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6968 , TFLOPS: 102.51178975541684, Tokens per sec: 83765.02229738148, Loss: 2.2631118297576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6969 , TFLOPS: 102.68261362962275, Tokens per sec: 83904.60688239287, Loss: 2.2689428329467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6970 , TFLOPS: 102.61667414374597, Tokens per sec: 83850.72603104956, Loss: 2.2658355236053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6971 , TFLOPS: 103.41184056590973, Tokens per sec: 84500.47698400429, Loss: 2.281327486038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6972 , TFLOPS: 101.38971338864452, Tokens per sec: 82848.14481327646, Loss: 2.27262020111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6973 , TFLOPS: 102.78840396784372, Tokens per sec: 83991.0508910388, Loss: 2.271980047225952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6974 , TFLOPS: 102.62324465108438, Tokens per sec: 83856.09496173568, Loss: 2.277578115463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6975 , TFLOPS: 101.2242492817751, Tokens per sec: 82712.93983212842, Loss: 2.2756142616271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6976 , TFLOPS: 102.80241816162747, Tokens per sec: 84002.50224953693, Loss: 2.2744908332824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6977 , TFLOPS: 102.71797460643056, Tokens per sec: 83933.50124681504, Loss: 2.289776086807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6978 , TFLOPS: 101.50944674315883, Tokens per sec: 82945.98201946086, Loss: 2.275160789489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6979 , TFLOPS: 99.49765784231992, Tokens per sec: 81302.0975205319, Loss: 2.265864610671997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6980 , TFLOPS: 101.53804529683124, Tokens per sec: 82969.35063385886, Loss: 2.2578365802764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6981 , TFLOPS: 102.70137793778373, Tokens per sec: 83919.93968162745, Loss: 2.2789578437805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6982 , TFLOPS: 102.10012960971329, Tokens per sec: 83428.64419525223, Loss: 2.272008180618286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6983 , TFLOPS: 100.441976821691, Tokens per sec: 82073.72486750911, Loss: 2.254207134246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6984 , TFLOPS: 103.4818827294437, Tokens per sec: 84557.71023887358, Loss: 2.2876033782958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6985 , TFLOPS: 100.82370587345157, Tokens per sec: 82385.64550228266, Loss: 2.2778797149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6986 , TFLOPS: 100.82858263347168, Tokens per sec: 82389.63042843407, Loss: 2.284898281097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6987 , TFLOPS: 102.70253309871846, Tokens per sec: 83920.88359336372, Loss: 2.261955976486206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6988 , TFLOPS: 100.97765478216733, Tokens per sec: 82511.44111859179, Loss: 2.2750277519226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6989 , TFLOPS: 102.65014048979565, Tokens per sec: 83878.07224390714, Loss: 2.2736213207244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6990 , TFLOPS: 101.02697384553896, Tokens per sec: 82551.74099485837, Loss: 2.29495906829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6991 , TFLOPS: 101.61477864662477, Tokens per sec: 83032.05142926729, Loss: 2.258714199066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6992 , TFLOPS: 102.18111304194356, Tokens per sec: 83494.81784242658, Loss: 2.275515556335449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6993 , TFLOPS: 101.91467584394131, Tokens per sec: 83277.10514923529, Loss: 2.2691330909729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6994 , TFLOPS: 102.67863437442492, Tokens per sec: 83901.35533053557, Loss: 2.276376485824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6995 , TFLOPS: 101.537223024238, Tokens per sec: 82968.67873375575, Loss: 2.2772860527038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6996 , TFLOPS: 102.1843186025288, Tokens per sec: 83497.43718849924, Loss: 2.2941370010375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6997 , TFLOPS: 102.35654473497516, Tokens per sec: 83638.16759481624, Loss: 2.288391590118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6998 , TFLOPS: 102.7968484297143, Tokens per sec: 83997.95107821302, Loss: 2.290632963180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6999 , TFLOPS: 102.19658768985943, Tokens per sec: 83507.4625755915, Loss: 2.2668955326080322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7000 , TFLOPS: 101.32389774931254, Tokens per sec: 82794.36516013302, Loss: 2.2814724445343018 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/model.pt +[rank0]:[2024-08-28 05:08:14,133] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00764481199439615, 'preprocessing_with_comm': 0.0016291560023091733, 'state_converting': 3.3176182819879614, : 3.328515701025026}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000, took 15.60s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005000 + eval ppl=7.839247703552246, eval loss=2.059142827987671 +------------------------------------------------------------------ +iteration: 7001 , TFLOPS: 102.5780345493752, Tokens per sec: 83819.15262381756, Loss: 2.2997405529022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7002 , TFLOPS: 100.76999834718957, Tokens per sec: 82341.75970001926, Loss: 2.2685577869415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7003 , TFLOPS: 101.27585215518063, Tokens per sec: 82755.10586836423, Loss: 2.2696893215179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7004 , TFLOPS: 101.55482484563927, Tokens per sec: 82983.06163513391, Loss: 2.2769618034362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7005 , TFLOPS: 100.34374720058258, Tokens per sec: 81993.45891544601, Loss: 2.2783048152923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7006 , TFLOPS: 102.34823272344556, Tokens per sec: 83631.37563622519, Loss: 2.292407751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7007 , TFLOPS: 101.78839233361381, Tokens per sec: 83173.91564211994, Loss: 2.288767099380493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7008 , TFLOPS: 101.83213849006725, Tokens per sec: 83209.6617526843, Loss: 2.283416509628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7009 , TFLOPS: 102.80456005418495, Tokens per sec: 84004.25244508285, Loss: 2.2718346118927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7010 , TFLOPS: 102.59067214414881, Tokens per sec: 83829.47912782771, Loss: 2.276986598968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7011 , TFLOPS: 102.80848856659824, Tokens per sec: 84007.46253370467, Loss: 2.278374195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7012 , TFLOPS: 103.47210841544981, Tokens per sec: 84549.72339529595, Loss: 2.262369155883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7013 , TFLOPS: 101.44904010657658, Tokens per sec: 82896.62220170435, Loss: 2.2717206478118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7014 , TFLOPS: 103.19264478710669, Tokens per sec: 84321.366470543, Loss: 2.2538070678710938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7015 , TFLOPS: 101.67458161528315, Tokens per sec: 83080.91797442343, Loss: 2.2724356651306152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7016 , TFLOPS: 102.71466726506736, Tokens per sec: 83930.79873304864, Loss: 2.285609245300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7017 , TFLOPS: 102.63989204715678, Tokens per sec: 83869.69797760854, Loss: 2.266453266143799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7018 , TFLOPS: 103.44386543661113, Tokens per sec: 84526.64533024792, Loss: 2.2564895153045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7019 , TFLOPS: 102.04664602659412, Tokens per sec: 83384.94138269563, Loss: 2.2760775089263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7020 , TFLOPS: 103.4217551076719, Tokens per sec: 84508.57841130118, Loss: 2.2704029083251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7021 , TFLOPS: 102.28986836922007, Tokens per sec: 83583.6846199554, Loss: 2.2785308361053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7022 , TFLOPS: 102.69295331755195, Tokens per sec: 83913.05570756705, Loss: 2.2589728832244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7023 , TFLOPS: 101.50415585466915, Tokens per sec: 82941.65869827635, Loss: 2.2858152389526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7024 , TFLOPS: 102.60958520878121, Tokens per sec: 83844.93347980444, Loss: 2.2651352882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7025 , TFLOPS: 102.69706414752719, Tokens per sec: 83916.414772562, Loss: 2.2724647521972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7026 , TFLOPS: 102.77723275613901, Tokens per sec: 83981.9226063823, Loss: 2.252869129180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7027 , TFLOPS: 101.56269127577026, Tokens per sec: 82989.48949769387, Loss: 2.2787415981292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7028 , TFLOPS: 103.45185026160743, Tokens per sec: 84533.16993630008, Loss: 2.2880914211273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7029 , TFLOPS: 102.09646835778746, Tokens per sec: 83425.6524920546, Loss: 2.282072067260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7030 , TFLOPS: 100.64239477237366, Tokens per sec: 82237.49153422893, Loss: 2.2750484943389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7031 , TFLOPS: 103.27132397400453, Tokens per sec: 84385.65725953967, Loss: 2.282566785812378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7032 , TFLOPS: 101.33125860554323, Tokens per sec: 82800.37990524442, Loss: 2.2673466205596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7033 , TFLOPS: 102.05082988042805, Tokens per sec: 83388.36011736533, Loss: 2.2481253147125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7034 , TFLOPS: 101.90185334067425, Tokens per sec: 83266.62755173519, Loss: 2.2727842330932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7035 , TFLOPS: 101.59323647578911, Tokens per sec: 83014.44876693266, Loss: 2.25130033493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7036 , TFLOPS: 101.43453887485465, Tokens per sec: 82884.77287196944, Loss: 2.276902675628662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7037 , TFLOPS: 101.58709490415931, Tokens per sec: 83009.43033065584, Loss: 2.274911642074585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7038 , TFLOPS: 100.11685695840787, Tokens per sec: 81808.06105789062, Loss: 2.2564783096313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7039 , TFLOPS: 102.73581041781942, Tokens per sec: 83948.07534743552, Loss: 2.280325412750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7040 , TFLOPS: 101.01689580218859, Tokens per sec: 82543.50596620487, Loss: 2.2949540615081787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7041 , TFLOPS: 102.26336932501792, Tokens per sec: 83562.03156879173, Loss: 2.263021945953369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7042 , TFLOPS: 100.8699916137922, Tokens per sec: 82423.46677221594, Loss: 2.2831015586853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7043 , TFLOPS: 101.07563424169196, Tokens per sec: 82591.50265718502, Loss: 2.2614099979400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7044 , TFLOPS: 101.66660041311188, Tokens per sec: 83074.39632867486, Loss: 2.2553179264068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7045 , TFLOPS: 101.99769303816706, Tokens per sec: 83344.94063568984, Loss: 2.277681827545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7046 , TFLOPS: 102.76204344833373, Tokens per sec: 83969.51103196703, Loss: 2.2822437286376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7047 , TFLOPS: 101.95479819575404, Tokens per sec: 83309.89015574269, Loss: 2.2617506980895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7048 , TFLOPS: 103.37027818380452, Tokens per sec: 84466.51529167521, Loss: 2.2871668338775635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7049 , TFLOPS: 102.72097216270248, Tokens per sec: 83935.95062720879, Loss: 2.257983684539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7050 , TFLOPS: 103.42223368624407, Tokens per sec: 84508.9694701722, Loss: 2.271122455596924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7051 , TFLOPS: 101.34932133036088, Tokens per sec: 82815.13942266883, Loss: 2.2721238136291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7052 , TFLOPS: 103.50360182643945, Tokens per sec: 84575.45747212811, Loss: 2.2712998390197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7053 , TFLOPS: 101.85563151246329, Tokens per sec: 83228.85850604829, Loss: 2.254079818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7054 , TFLOPS: 102.72701211809103, Tokens per sec: 83940.88603023907, Loss: 2.2785439491271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7055 , TFLOPS: 102.14181064826948, Tokens per sec: 83462.70284482186, Loss: 2.262328863143921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7056 , TFLOPS: 102.60117690605146, Tokens per sec: 83838.06283919499, Loss: 2.284252882003784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7057 , TFLOPS: 102.17394083580376, Tokens per sec: 83488.95724816073, Loss: 2.271228075027466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7058 , TFLOPS: 103.21017237174752, Tokens per sec: 84335.68871115318, Loss: 2.277047634124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7059 , TFLOPS: 102.64975279417956, Tokens per sec: 83877.75544783907, Loss: 2.2690391540527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7060 , TFLOPS: 102.79985053208466, Tokens per sec: 84000.40417334084, Loss: 2.2781219482421875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7061 , TFLOPS: 101.53638373009773, Tokens per sec: 82967.99292490837, Loss: 2.2755441665649414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7062 , TFLOPS: 103.30245514489442, Tokens per sec: 84411.09534066144, Loss: 2.288379669189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7063 , TFLOPS: 102.54799288061463, Tokens per sec: 83794.60480293194, Loss: 2.287290096282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7064 , TFLOPS: 102.7134887416296, Tokens per sec: 83929.83573120974, Loss: 2.274745225906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7065 , TFLOPS: 100.80856276025078, Tokens per sec: 82373.27167466788, Loss: 2.273499011993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7066 , TFLOPS: 103.35574789066894, Tokens per sec: 84454.64221511112, Loss: 2.2834601402282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7067 , TFLOPS: 102.08844608213126, Tokens per sec: 83419.09728410418, Loss: 2.2708280086517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7068 , TFLOPS: 101.63345747273782, Tokens per sec: 83047.31437892003, Loss: 2.2559895515441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7069 , TFLOPS: 103.38126046509514, Tokens per sec: 84475.48919642686, Loss: 2.270322799682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7070 , TFLOPS: 101.25251107268618, Tokens per sec: 82736.03326900509, Loss: 2.2731823921203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7071 , TFLOPS: 101.15164449589649, Tokens per sec: 82653.61259257355, Loss: 2.2544643878936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7072 , TFLOPS: 102.05494050528814, Tokens per sec: 83391.71901475539, Loss: 2.279672384262085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7073 , TFLOPS: 102.14258751699354, Tokens per sec: 83463.33764425467, Loss: 2.2821075916290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7074 , TFLOPS: 101.3973622989039, Tokens per sec: 82854.39493474987, Loss: 2.273376703262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7075 , TFLOPS: 102.08540311158914, Tokens per sec: 83416.6107945412, Loss: 2.2598624229431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7076 , TFLOPS: 100.04848618975355, Tokens per sec: 81752.19354280301, Loss: 2.296133279800415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7077 , TFLOPS: 102.7029782608635, Tokens per sec: 83921.24734681191, Loss: 2.291795253753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7078 , TFLOPS: 100.86690204855788, Tokens per sec: 82420.9422089305, Loss: 2.277677536010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7079 , TFLOPS: 102.26237048147416, Tokens per sec: 83561.21538801961, Loss: 2.2712767124176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7080 , TFLOPS: 100.87929056858215, Tokens per sec: 82431.06518755121, Loss: 2.2692644596099854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7081 , TFLOPS: 100.73664986888349, Tokens per sec: 82314.50979992916, Loss: 2.2857210636138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7082 , TFLOPS: 101.81508083144188, Tokens per sec: 83195.7235007183, Loss: 2.2930359840393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7083 , TFLOPS: 101.49519145577777, Tokens per sec: 82934.33365717805, Loss: 2.2859222888946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7084 , TFLOPS: 102.76627858063897, Tokens per sec: 83972.97166759564, Loss: 2.268291473388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7085 , TFLOPS: 102.8094970604001, Tokens per sec: 84008.28659995113, Loss: 2.285736083984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7086 , TFLOPS: 102.65893327974595, Tokens per sec: 83885.2570589221, Loss: 2.2757761478424072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7087 , TFLOPS: 103.45086510577872, Tokens per sec: 84532.36494011227, Loss: 2.2843587398529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7088 , TFLOPS: 102.71423003745359, Tokens per sec: 83930.44146311017, Loss: 2.273698329925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7089 , TFLOPS: 102.16103486564667, Tokens per sec: 83478.41144771615, Loss: 2.27634334564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7090 , TFLOPS: 103.37289694270113, Tokens per sec: 84468.65514698249, Loss: 2.2726383209228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7091 , TFLOPS: 101.91663794222845, Tokens per sec: 83278.70843025461, Loss: 2.2672085762023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7092 , TFLOPS: 102.71657052042293, Tokens per sec: 83932.35393199383, Loss: 2.2806100845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7093 , TFLOPS: 101.95801681002861, Tokens per sec: 83312.52016832095, Loss: 2.2654969692230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7094 , TFLOPS: 102.63085957103681, Tokens per sec: 83862.3173088543, Loss: 2.2620601654052734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7095 , TFLOPS: 102.05621099106065, Tokens per sec: 83392.75716138532, Loss: 2.293680191040039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7096 , TFLOPS: 102.83898548682428, Tokens per sec: 84032.38235228194, Loss: 2.2823328971862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7097 , TFLOPS: 102.71370795305113, Tokens per sec: 83930.01485450528, Loss: 2.2640533447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7098 , TFLOPS: 102.16738803312025, Tokens per sec: 83483.60278440388, Loss: 2.275115489959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7099 , TFLOPS: 101.5104363746353, Tokens per sec: 82946.79067281567, Loss: 2.269008159637451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7100 , TFLOPS: 103.34715670633518, Tokens per sec: 84447.62213723526, Loss: 2.2819292545318604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7101 , TFLOPS: 102.60305634718226, Tokens per sec: 83839.5985789241, Loss: 2.262014150619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7102 , TFLOPS: 102.82567702486661, Tokens per sec: 84021.50766542603, Loss: 2.2742583751678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7103 , TFLOPS: 101.77524212422117, Tokens per sec: 83163.1702675087, Loss: 2.2719955444335938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7104 , TFLOPS: 103.42668592288034, Tokens per sec: 84512.60750733846, Loss: 2.2502942085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7105 , TFLOPS: 101.99890465758843, Tokens per sec: 83345.9306811089, Loss: 2.2550339698791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7106 , TFLOPS: 101.70940883156268, Tokens per sec: 83109.37618937774, Loss: 2.2814526557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7107 , TFLOPS: 102.64784921847811, Tokens per sec: 83876.19998713101, Loss: 2.278510808944702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7108 , TFLOPS: 101.40753862214264, Tokens per sec: 82862.7102704301, Loss: 2.2780697345733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7109 , TFLOPS: 101.86371701166118, Tokens per sec: 83235.46537558215, Loss: 2.2809245586395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7110 , TFLOPS: 101.96726281562736, Tokens per sec: 83320.07531750905, Loss: 2.278830051422119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7111 , TFLOPS: 102.29097770024899, Tokens per sec: 83584.59108289584, Loss: 2.2920539379119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7112 , TFLOPS: 101.49114236858286, Tokens per sec: 82931.0250437983, Loss: 2.2643752098083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7113 , TFLOPS: 101.3340912182642, Tokens per sec: 82802.69450601668, Loss: 2.292055606842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7114 , TFLOPS: 100.79665527910532, Tokens per sec: 82363.5417652981, Loss: 2.261723756790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7115 , TFLOPS: 102.72809426273136, Tokens per sec: 83941.77027848155, Loss: 2.266064167022705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7116 , TFLOPS: 100.98723075836273, Tokens per sec: 82519.2658952486, Loss: 2.2741992473602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7117 , TFLOPS: 102.36796028998161, Tokens per sec: 83647.49553867453, Loss: 2.2880375385284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7118 , TFLOPS: 100.8843698545204, Tokens per sec: 82435.2156028438, Loss: 2.282773017883301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7119 , TFLOPS: 100.90144286450649, Tokens per sec: 82449.16639880181, Loss: 2.269138813018799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7120 , TFLOPS: 102.31267985330084, Tokens per sec: 83602.32447081762, Loss: 2.299452304840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7121 , TFLOPS: 101.42818292322265, Tokens per sec: 82879.57926027419, Loss: 2.274435043334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7122 , TFLOPS: 102.76037116356946, Tokens per sec: 83968.14456503771, Loss: 2.2774593830108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7123 , TFLOPS: 102.99105076454084, Tokens per sec: 84156.63880521264, Loss: 2.269136428833008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7124 , TFLOPS: 103.4119544177982, Tokens per sec: 84500.57001531316, Loss: 2.294015884399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7125 , TFLOPS: 102.67796147942082, Tokens per sec: 83900.80549070585, Loss: 2.268717050552368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7126 , TFLOPS: 103.38711198860227, Tokens per sec: 84480.27062691616, Loss: 2.2825117111206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7127 , TFLOPS: 101.26556619301114, Tokens per sec: 82746.70094389131, Loss: 2.274770498275757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7128 , TFLOPS: 103.40699948041468, Tokens per sec: 84496.52120842566, Loss: 2.2702953815460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7129 , TFLOPS: 102.60594804602542, Tokens per sec: 83841.96146048779, Loss: 2.27484393119812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7130 , TFLOPS: 102.04808002711273, Tokens per sec: 83386.11314143385, Loss: 2.2782726287841797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7131 , TFLOPS: 101.4850692730399, Tokens per sec: 82926.06256109478, Loss: 2.2787563800811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7132 , TFLOPS: 102.53187547037439, Tokens per sec: 83781.43485212559, Loss: 2.2722768783569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7133 , TFLOPS: 103.36858712238343, Tokens per sec: 84465.13348185626, Loss: 2.2599711418151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7134 , TFLOPS: 102.13148495492547, Tokens per sec: 83454.26545498348, Loss: 2.266045570373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7135 , TFLOPS: 102.08377924405616, Tokens per sec: 83415.28389057792, Loss: 2.2912442684173584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7136 , TFLOPS: 102.06701117130372, Tokens per sec: 83401.58226668331, Loss: 2.2769200801849365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7137 , TFLOPS: 102.04643473791901, Tokens per sec: 83384.76873328014, Loss: 2.2873306274414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7138 , TFLOPS: 102.15270245285492, Tokens per sec: 83471.60281872863, Loss: 2.28183913230896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7139 , TFLOPS: 102.04532217620893, Tokens per sec: 83383.85963046695, Loss: 2.294166088104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7140 , TFLOPS: 101.46971418681426, Tokens per sec: 82913.51552486472, Loss: 2.273002862930298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7141 , TFLOPS: 102.74362229352823, Tokens per sec: 83954.45863217275, Loss: 2.271786689758301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7142 , TFLOPS: 102.20378905277533, Tokens per sec: 83513.34699460967, Loss: 2.257168769836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7143 , TFLOPS: 102.68783173658211, Tokens per sec: 83908.87073191538, Loss: 2.2941107749938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7144 , TFLOPS: 100.98300406985288, Tokens per sec: 82515.81215926255, Loss: 2.2789599895477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7145 , TFLOPS: 103.23938877099279, Tokens per sec: 84359.56218307352, Loss: 2.291100025177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7146 , TFLOPS: 101.39685037805472, Tokens per sec: 82853.976631046, Loss: 2.291269540786743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7147 , TFLOPS: 101.47434245264819, Tokens per sec: 82917.29740001979, Loss: 2.2685556411743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7148 , TFLOPS: 101.22833137320822, Tokens per sec: 82716.27541412081, Loss: 2.246782064437866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7149 , TFLOPS: 101.83022536769265, Tokens per sec: 83208.09849114386, Loss: 2.2726545333862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7150 , TFLOPS: 101.54452834722998, Tokens per sec: 82974.64810122823, Loss: 2.284428596496582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7151 , TFLOPS: 102.18961879910655, Tokens per sec: 83501.76811556214, Loss: 2.2810614109039307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7152 , TFLOPS: 102.09841523946345, Tokens per sec: 83427.24333918926, Loss: 2.273655891418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7153 , TFLOPS: 102.77053509793923, Tokens per sec: 83976.4497774542, Loss: 2.2919082641601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7154 , TFLOPS: 101.13882807952204, Tokens per sec: 82643.13996883009, Loss: 2.2795238494873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7155 , TFLOPS: 102.72299863124366, Tokens per sec: 83937.60650681972, Loss: 2.3012499809265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7156 , TFLOPS: 101.95283874520685, Tokens per sec: 83308.28903825984, Loss: 2.2620625495910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7157 , TFLOPS: 102.06180788075257, Tokens per sec: 83397.33052402934, Loss: 2.2602627277374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7158 , TFLOPS: 102.4972724528174, Tokens per sec: 83753.15983571886, Loss: 2.2722930908203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7159 , TFLOPS: 100.25265935563804, Tokens per sec: 81919.0286925321, Loss: 2.2618658542633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7160 , TFLOPS: 102.8725135096373, Tokens per sec: 84059.77896281055, Loss: 2.274928331375122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7161 , TFLOPS: 102.76287192597502, Tokens per sec: 83970.18800237463, Loss: 2.2697038650512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7162 , TFLOPS: 103.42132673513858, Tokens per sec: 84508.2283770769, Loss: 2.2746682167053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7163 , TFLOPS: 102.07176005871541, Tokens per sec: 83405.46270483463, Loss: 2.294114351272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7164 , TFLOPS: 103.42424762732033, Tokens per sec: 84510.61511326906, Loss: 2.2735955715179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7165 , TFLOPS: 102.75279603021485, Tokens per sec: 83961.95472857238, Loss: 2.2862327098846436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7166 , TFLOPS: 102.08387647613063, Tokens per sec: 83415.36334140894, Loss: 2.264540195465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7167 , TFLOPS: 102.51653928521876, Tokens per sec: 83768.9032604464, Loss: 2.300320625305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7168 , TFLOPS: 102.59448421808587, Tokens per sec: 83832.59407157318, Loss: 2.2863881587982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7169 , TFLOPS: 100.60793974678545, Tokens per sec: 82209.33744586953, Loss: 2.2656610012054443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7170 , TFLOPS: 103.40408197301025, Tokens per sec: 84494.13723802217, Loss: 2.280975103378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7171 , TFLOPS: 102.6873669575124, Tokens per sec: 83908.49094897303, Loss: 2.3020081520080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7172 , TFLOPS: 102.15313674546866, Tokens per sec: 83471.957690403, Loss: 2.2844581604003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7173 , TFLOPS: 102.59550802703063, Tokens per sec: 83833.43065221715, Loss: 2.282285213470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7174 , TFLOPS: 101.38303474176954, Tokens per sec: 82842.68751899125, Loss: 2.2687535285949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7175 , TFLOPS: 102.186743007193, Tokens per sec: 83499.41823196008, Loss: 2.271693229675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7176 , TFLOPS: 102.08949828884707, Tokens per sec: 83419.95706929787, Loss: 2.260793685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7177 , TFLOPS: 101.45311356636465, Tokens per sec: 82899.95073055735, Loss: 2.263770580291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7178 , TFLOPS: 101.5106997559916, Tokens per sec: 82947.00588850194, Loss: 2.2773025035858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7179 , TFLOPS: 102.27500714831046, Tokens per sec: 83571.5411337883, Loss: 2.2864205837249756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7180 , TFLOPS: 102.20253054205159, Tokens per sec: 83512.31863310035, Loss: 2.273921012878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7181 , TFLOPS: 101.22403711565443, Tokens per sec: 82712.76646572958, Loss: 2.277338743209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7182 , TFLOPS: 101.70412976550571, Tokens per sec: 83105.06252860733, Loss: 2.2748422622680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7183 , TFLOPS: 102.9040651919269, Tokens per sec: 84085.56065462195, Loss: 2.272613286972046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7184 , TFLOPS: 102.13193142360083, Tokens per sec: 83454.63027603125, Loss: 2.249999761581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7185 , TFLOPS: 101.43717069611004, Tokens per sec: 82886.92340086632, Loss: 2.301072597503662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7186 , TFLOPS: 102.62383234630997, Tokens per sec: 83856.57518263314, Loss: 2.2787158489227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7187 , TFLOPS: 102.31549733080045, Tokens per sec: 83604.6267042109, Loss: 2.2689192295074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7188 , TFLOPS: 101.55818599721566, Tokens per sec: 82985.80811861072, Loss: 2.261141538619995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7189 , TFLOPS: 102.90184611389532, Tokens per sec: 84083.74738883818, Loss: 2.2841243743896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7190 , TFLOPS: 100.69234947634479, Tokens per sec: 82278.31080879222, Loss: 2.2687132358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7191 , TFLOPS: 102.70904359609851, Tokens per sec: 83926.20348836806, Loss: 2.275585889816284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7192 , TFLOPS: 101.8991254848993, Tokens per sec: 83264.3985505602, Loss: 2.2668604850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7193 , TFLOPS: 102.7191757331488, Tokens per sec: 83934.48271837627, Loss: 2.2874045372009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7194 , TFLOPS: 101.29373447221198, Tokens per sec: 82769.71796994234, Loss: 2.2706847190856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7195 , TFLOPS: 101.29009118859285, Tokens per sec: 82766.74094911068, Loss: 2.238954544067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7196 , TFLOPS: 102.26937829891587, Tokens per sec: 83566.9416560485, Loss: 2.272683620452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7197 , TFLOPS: 101.48397742373459, Tokens per sec: 82925.1703829207, Loss: 2.2549850940704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7198 , TFLOPS: 101.8293330911449, Tokens per sec: 83207.36938900745, Loss: 2.2805771827697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7199 , TFLOPS: 102.78711781888275, Tokens per sec: 83989.99994561436, Loss: 2.2861673831939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7200 , TFLOPS: 103.40429060347947, Tokens per sec: 84494.30771534919, Loss: 2.2784435749053955 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/model.pt +[rank0]:[2024-08-28 06:00:58,177] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007793828990543261, 'preprocessing_with_comm': 0.0016115089820232242, 'state_converting': 3.31170370298787, : 3.3228086949966382}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200, took 15.67s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005200 + eval ppl=7.697409152984619, eval loss=2.040883779525757 +------------------------------------------------------------------ +iteration: 7201 , TFLOPS: 100.9423982765935, Tokens per sec: 82482.63212030419, Loss: 2.279238700866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7202 , TFLOPS: 101.37170402287502, Tokens per sec: 82833.42889690389, Loss: 2.254477024078369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7203 , TFLOPS: 101.42251658848488, Tokens per sec: 82874.94915229552, Loss: 2.276909589767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7204 , TFLOPS: 100.41592162485962, Tokens per sec: 82052.43449545774, Loss: 2.2612993717193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7205 , TFLOPS: 101.49438778946583, Tokens per sec: 82933.67696074636, Loss: 2.264246940612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7206 , TFLOPS: 102.87546885088004, Tokens per sec: 84062.19384821724, Loss: 2.2987074851989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7207 , TFLOPS: 101.50803636266144, Tokens per sec: 82944.82956125002, Loss: 2.2807888984680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7208 , TFLOPS: 101.536640407621, Tokens per sec: 82968.2026627201, Loss: 2.281811237335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7209 , TFLOPS: 102.74601432794601, Tokens per sec: 83956.41322507196, Loss: 2.262594699859619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7210 , TFLOPS: 103.38499127969297, Tokens per sec: 84478.53774107445, Loss: 2.2747750282287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7211 , TFLOPS: 103.42443140358475, Tokens per sec: 84510.76528158554, Loss: 2.258927345275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7212 , TFLOPS: 103.44603418176408, Tokens per sec: 84528.41746774086, Loss: 2.287552833557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7213 , TFLOPS: 101.87315599426714, Tokens per sec: 83243.17821125052, Loss: 2.2553460597991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7214 , TFLOPS: 102.50499015859687, Tokens per sec: 83759.4661717827, Loss: 2.266462802886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7215 , TFLOPS: 103.29674924007396, Tokens per sec: 84406.43289894958, Loss: 2.2699451446533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7216 , TFLOPS: 102.51165276453109, Tokens per sec: 83764.91035860227, Loss: 2.2492852210998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7217 , TFLOPS: 101.54047965078911, Tokens per sec: 82971.33980714367, Loss: 2.2756426334381104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7218 , TFLOPS: 103.36102291388232, Tokens per sec: 84458.95257236995, Loss: 2.283201217651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7219 , TFLOPS: 102.76181367210624, Tokens per sec: 83969.32327589659, Loss: 2.266850709915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7220 , TFLOPS: 102.0459861886953, Tokens per sec: 83384.40221216275, Loss: 2.2760818004608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7221 , TFLOPS: 103.4723644708675, Tokens per sec: 84549.93262476921, Loss: 2.254586696624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7222 , TFLOPS: 100.84350871755404, Tokens per sec: 82401.82691596934, Loss: 2.29972243309021 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7223 , TFLOPS: 102.756380513109, Tokens per sec: 83964.88370181799, Loss: 2.274655342102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7224 , TFLOPS: 100.64390517585059, Tokens per sec: 82238.72572379117, Loss: 2.277001142501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7225 , TFLOPS: 102.73219221055244, Tokens per sec: 83945.11881713672, Loss: 2.2913620471954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7226 , TFLOPS: 101.86328654620749, Tokens per sec: 83235.11363117878, Loss: 2.29305100440979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7227 , TFLOPS: 101.64520524850089, Tokens per sec: 83056.91378891068, Loss: 2.286384105682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7228 , TFLOPS: 101.74169530691825, Tokens per sec: 83135.75829951858, Loss: 2.296285629272461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7229 , TFLOPS: 103.34577950160795, Tokens per sec: 84446.49678780022, Loss: 2.2767796516418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7230 , TFLOPS: 100.31468901792297, Tokens per sec: 81969.71472637069, Loss: 2.2594265937805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7231 , TFLOPS: 101.99871661437666, Tokens per sec: 83345.77702615992, Loss: 2.2767186164855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7232 , TFLOPS: 102.7318090282387, Tokens per sec: 83944.80570900417, Loss: 2.2677860260009766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7233 , TFLOPS: 100.21385841134126, Tokens per sec: 81887.32344212204, Loss: 2.275294303894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7234 , TFLOPS: 102.70306600098489, Tokens per sec: 83921.31904152373, Loss: 2.274444103240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7235 , TFLOPS: 102.05565907321, Tokens per sec: 83392.30617510193, Loss: 2.262493371963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7236 , TFLOPS: 101.37060228512001, Tokens per sec: 82832.52863862303, Loss: 2.252906084060669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7237 , TFLOPS: 102.84760546775676, Tokens per sec: 84039.42596059993, Loss: 2.2855958938598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7238 , TFLOPS: 100.89714094181964, Tokens per sec: 82445.65118703304, Loss: 2.255380153656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7239 , TFLOPS: 102.69664160625955, Tokens per sec: 83916.06950321503, Loss: 2.298956871032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7240 , TFLOPS: 101.41149301495192, Tokens per sec: 82865.9415065897, Loss: 2.251894235610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7241 , TFLOPS: 101.16683503911779, Tokens per sec: 82666.02517647904, Loss: 2.2644131183624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7242 , TFLOPS: 101.60195889862237, Tokens per sec: 83021.57608316487, Loss: 2.270488977432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7243 , TFLOPS: 101.56633208881902, Tokens per sec: 82992.46449975888, Loss: 2.28086519241333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7244 , TFLOPS: 102.72012435679918, Tokens per sec: 83935.25786318074, Loss: 2.283867120742798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7245 , TFLOPS: 101.85213231833838, Tokens per sec: 83225.99922445156, Loss: 2.2925970554351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7246 , TFLOPS: 100.93003192658207, Tokens per sec: 82472.52725737171, Loss: 2.2646312713623047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7247 , TFLOPS: 102.11802339115074, Tokens per sec: 83443.26566469167, Loss: 2.280150890350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7248 , TFLOPS: 103.4921939027725, Tokens per sec: 84566.13576403377, Loss: 2.268836259841919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7249 , TFLOPS: 103.37162610671447, Tokens per sec: 84467.61671418358, Loss: 2.28361177444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7250 , TFLOPS: 103.35134169444328, Tokens per sec: 84451.04179874955, Loss: 2.278639078140259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7251 , TFLOPS: 102.73638023438733, Tokens per sec: 83948.54095922154, Loss: 2.2832581996917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7252 , TFLOPS: 102.64223794899931, Tokens per sec: 83871.61487439295, Loss: 2.280543327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7253 , TFLOPS: 103.21636521000008, Tokens per sec: 84340.74903870719, Loss: 2.284991979598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7254 , TFLOPS: 101.3500526366156, Tokens per sec: 82815.73699183506, Loss: 2.2866923809051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7255 , TFLOPS: 101.53636790244354, Tokens per sec: 82967.9799917247, Loss: 2.2718591690063477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7256 , TFLOPS: 103.37416953425112, Tokens per sec: 84469.69501429741, Loss: 2.283615827560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7257 , TFLOPS: 102.86025479792823, Tokens per sec: 84049.76205390575, Loss: 2.273437261581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7258 , TFLOPS: 102.17191605771788, Tokens per sec: 83487.30274986434, Loss: 2.2689454555511475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7259 , TFLOPS: 102.69245829252965, Tokens per sec: 83912.65120987826, Loss: 2.254443645477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7260 , TFLOPS: 100.52593493083015, Tokens per sec: 82142.3291997607, Loss: 2.273494243621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7261 , TFLOPS: 102.65676045294164, Tokens per sec: 83883.48158620675, Loss: 2.279557466506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7262 , TFLOPS: 102.15858557069888, Tokens per sec: 83476.41006576385, Loss: 2.2849555015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7263 , TFLOPS: 101.92848948725194, Tokens per sec: 83288.39263277923, Loss: 2.2756762504577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7264 , TFLOPS: 102.2916211920771, Tokens per sec: 83585.11689663355, Loss: 2.2690958976745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7265 , TFLOPS: 100.95625784567815, Tokens per sec: 82493.95713098031, Loss: 2.2945854663848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7266 , TFLOPS: 102.05251140787023, Tokens per sec: 83389.73413672468, Loss: 2.2657899856567383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7267 , TFLOPS: 102.60292256744502, Tokens per sec: 83839.48926405699, Loss: 2.274125099182129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7268 , TFLOPS: 100.41661641826073, Tokens per sec: 82053.00222903138, Loss: 2.29594349861145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7269 , TFLOPS: 101.5226122955696, Tokens per sec: 82956.73993124717, Loss: 2.270930767059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7270 , TFLOPS: 102.85661008230186, Tokens per sec: 84046.78386294412, Loss: 2.283933639526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7271 , TFLOPS: 99.27867656406656, Tokens per sec: 81123.16228099147, Loss: 2.2714643478393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7272 , TFLOPS: 103.46769458631299, Tokens per sec: 84546.11674188619, Loss: 2.286557674407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7273 , TFLOPS: 100.78288919737422, Tokens per sec: 82352.29314554513, Loss: 2.2922403812408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7274 , TFLOPS: 102.09817333979632, Tokens per sec: 83427.0456767442, Loss: 2.2985341548919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7275 , TFLOPS: 101.50392110701742, Tokens per sec: 82941.46687992717, Loss: 2.2925429344177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7276 , TFLOPS: 100.38870278254916, Tokens per sec: 82030.19327873054, Loss: 2.271320104598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7277 , TFLOPS: 102.68404228675453, Tokens per sec: 83905.77427491214, Loss: 2.2727973461151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7278 , TFLOPS: 100.84284093818417, Tokens per sec: 82401.28125625606, Loss: 2.271914482116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7279 , TFLOPS: 101.55833593297211, Tokens per sec: 82985.930634977, Loss: 2.279122829437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7280 , TFLOPS: 101.56123512134396, Tokens per sec: 82988.29963642739, Loss: 2.2542262077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7281 , TFLOPS: 101.70304699624738, Tokens per sec: 83104.1777699734, Loss: 2.2755393981933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7282 , TFLOPS: 103.36891688001387, Tokens per sec: 84465.40293530481, Loss: 2.2583718299865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7283 , TFLOPS: 101.01903847006564, Tokens per sec: 82545.25679528438, Loss: 2.2734649181365967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7284 , TFLOPS: 101.7103670584521, Tokens per sec: 83110.15918123603, Loss: 2.2679054737091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7285 , TFLOPS: 102.65786287647204, Tokens per sec: 83884.38240485234, Loss: 2.2827110290527344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7286 , TFLOPS: 102.6373044685249, Tokens per sec: 83867.58360049823, Loss: 2.2619822025299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7287 , TFLOPS: 102.63960049158561, Tokens per sec: 83869.45974004603, Loss: 2.2545199394226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7288 , TFLOPS: 103.31768851731577, Tokens per sec: 84423.54292140882, Loss: 2.2835915088653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7289 , TFLOPS: 102.625456675119, Tokens per sec: 83857.90246351692, Loss: 2.285536766052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7290 , TFLOPS: 101.53034273856555, Tokens per sec: 82963.05667521845, Loss: 2.276524782180786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7291 , TFLOPS: 103.37613368220211, Tokens per sec: 84471.29997014983, Loss: 2.286100149154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7292 , TFLOPS: 102.14410900165387, Tokens per sec: 83464.58088853714, Loss: 2.269258499145508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7293 , TFLOPS: 101.37163851760916, Tokens per sec: 82833.37537086489, Loss: 2.263669729232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7294 , TFLOPS: 102.86857048081482, Tokens per sec: 84056.55701245711, Loss: 2.267221212387085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7295 , TFLOPS: 102.7023248334101, Tokens per sec: 83920.71341441902, Loss: 2.274980068206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7296 , TFLOPS: 100.73368009964163, Tokens per sec: 82312.08312503298, Loss: 2.2698874473571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7297 , TFLOPS: 103.32226175428204, Tokens per sec: 84427.27983105955, Loss: 2.295919418334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7298 , TFLOPS: 101.49657950489672, Tokens per sec: 82935.46786784471, Loss: 2.2803397178649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7299 , TFLOPS: 101.97234650441695, Tokens per sec: 83324.22933048481, Loss: 2.28070068359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7300 , TFLOPS: 101.47728502526758, Tokens per sec: 82919.70185185564, Loss: 2.2582144737243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7301 , TFLOPS: 101.4743935200234, Tokens per sec: 82917.33912848667, Loss: 2.2797930240631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7302 , TFLOPS: 103.35396940692509, Tokens per sec: 84453.18897026178, Loss: 2.2660694122314453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7303 , TFLOPS: 100.04203337386963, Tokens per sec: 81746.920780834, Loss: 2.2767367362976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7304 , TFLOPS: 102.77093083072508, Tokens per sec: 83976.77314090058, Loss: 2.3081326484680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7305 , TFLOPS: 102.136611286802, Tokens per sec: 83458.45431272332, Loss: 2.2939207553863525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7306 , TFLOPS: 101.59492897296411, Tokens per sec: 83015.83174994311, Loss: 2.2689120769500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7307 , TFLOPS: 101.41883180617819, Tokens per sec: 82871.938221818, Loss: 2.283389091491699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7308 , TFLOPS: 102.23632821921338, Tokens per sec: 83539.93558513885, Loss: 2.287180185317993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7309 , TFLOPS: 101.00068288629659, Tokens per sec: 82530.25797527193, Loss: 2.282954454421997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7310 , TFLOPS: 101.85484632535564, Tokens per sec: 83228.21690945012, Loss: 2.2816824913024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7311 , TFLOPS: 101.58091744945676, Tokens per sec: 83004.38257339656, Loss: 2.273083209991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7312 , TFLOPS: 102.22787735686194, Tokens per sec: 83533.03016796694, Loss: 2.272742509841919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7313 , TFLOPS: 101.48825734150857, Tokens per sec: 82928.6676139087, Loss: 2.261439085006714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7314 , TFLOPS: 101.1051071031643, Tokens per sec: 82615.58569099302, Loss: 2.2517800331115723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7315 , TFLOPS: 102.67685963545739, Tokens per sec: 83899.90514563935, Loss: 2.261000633239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7316 , TFLOPS: 101.56821974046886, Tokens per sec: 82994.00694851443, Loss: 2.2743375301361084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7317 , TFLOPS: 101.30532363390617, Tokens per sec: 82779.18777229422, Loss: 2.2652323246002197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7318 , TFLOPS: 101.99758419705871, Tokens per sec: 83344.8516988184, Loss: 2.2667829990386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7319 , TFLOPS: 101.28845425060254, Tokens per sec: 82765.40336494027, Loss: 2.2706620693206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7320 , TFLOPS: 102.70075064265185, Tokens per sec: 83919.42710262719, Loss: 2.3019866943359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7321 , TFLOPS: 101.39428276388686, Tokens per sec: 82851.87856741318, Loss: 2.286785125732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7322 , TFLOPS: 100.83515292158017, Tokens per sec: 82394.99917998178, Loss: 2.2854485511779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7323 , TFLOPS: 102.74676861870981, Tokens per sec: 83957.02957547223, Loss: 2.2888431549072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7324 , TFLOPS: 102.75897667080325, Tokens per sec: 83967.005089101, Loss: 2.2685317993164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7325 , TFLOPS: 102.05773260200098, Tokens per sec: 83394.00050885441, Loss: 2.2525832653045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7326 , TFLOPS: 103.43388782626177, Tokens per sec: 84518.49236798492, Loss: 2.285240650177002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7327 , TFLOPS: 103.3496251555413, Tokens per sec: 84449.63917062512, Loss: 2.2898190021514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7328 , TFLOPS: 100.81802951680116, Tokens per sec: 82381.00720514118, Loss: 2.281198024749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7329 , TFLOPS: 103.41726414889567, Tokens per sec: 84504.90873327755, Loss: 2.2974202632904053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7330 , TFLOPS: 102.91972087234566, Tokens per sec: 84098.35331410517, Loss: 2.2833900451660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7331 , TFLOPS: 100.74335615604735, Tokens per sec: 82319.98967980593, Loss: 2.2577970027923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7332 , TFLOPS: 102.21636683956766, Tokens per sec: 83523.6246279787, Loss: 2.272326707839966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7333 , TFLOPS: 102.01836248862631, Tokens per sec: 83361.83017573909, Loss: 2.2753264904022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7334 , TFLOPS: 102.77248354471297, Tokens per sec: 83978.0419034705, Loss: 2.273407459259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7335 , TFLOPS: 102.048008494996, Tokens per sec: 83386.05469069986, Loss: 2.2741856575012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7336 , TFLOPS: 101.51351476093386, Tokens per sec: 82949.30610150493, Loss: 2.2726612091064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7337 , TFLOPS: 100.66484558790836, Tokens per sec: 82255.83667353775, Loss: 2.2596845626831055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7338 , TFLOPS: 102.19963513270982, Tokens per sec: 83509.95271959271, Loss: 2.2431342601776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7339 , TFLOPS: 101.64214776019458, Tokens per sec: 83054.41543650885, Loss: 2.261979579925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7340 , TFLOPS: 102.79319609052841, Tokens per sec: 83994.96665784466, Loss: 2.307548999786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7341 , TFLOPS: 99.8584932941543, Tokens per sec: 81596.94545695663, Loss: 2.3027775287628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7342 , TFLOPS: 102.72666709401003, Tokens per sec: 83940.60410218075, Loss: 2.258667230606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7343 , TFLOPS: 102.79652609617277, Tokens per sec: 83997.68769117864, Loss: 2.268885374069214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7344 , TFLOPS: 101.62529945918544, Tokens per sec: 83040.64825604044, Loss: 2.26723313331604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7345 , TFLOPS: 102.73298881998761, Tokens per sec: 83945.76974721275, Loss: 2.2923364639282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7346 , TFLOPS: 101.57653170983949, Tokens per sec: 83000.7988726563, Loss: 2.262726306915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7347 , TFLOPS: 101.57611973957819, Tokens per sec: 83000.46224115073, Loss: 2.287480115890503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7348 , TFLOPS: 102.76982564203352, Tokens per sec: 83975.87006277063, Loss: 2.2634594440460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7349 , TFLOPS: 100.66031784641456, Tokens per sec: 82252.13693940872, Loss: 2.2605385780334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7350 , TFLOPS: 102.05174364682931, Tokens per sec: 83389.10677941421, Loss: 2.272143602371216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7351 , TFLOPS: 102.8177726447171, Tokens per sec: 84015.04879292902, Loss: 2.2882802486419678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7352 , TFLOPS: 100.77089536644465, Tokens per sec: 82342.49267754391, Loss: 2.2744696140289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7353 , TFLOPS: 102.84929369021462, Tokens per sec: 84040.80545062895, Loss: 2.2753498554229736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7354 , TFLOPS: 100.3648518563106, Tokens per sec: 82010.70407292355, Loss: 2.2533295154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7355 , TFLOPS: 103.38705337937152, Tokens per sec: 84480.22273580497, Loss: 2.271648645401001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7356 , TFLOPS: 100.69378765430011, Tokens per sec: 82279.48598102153, Loss: 2.2796854972839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7357 , TFLOPS: 101.03488286281114, Tokens per sec: 82558.2036564677, Loss: 2.2662782669067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7358 , TFLOPS: 102.71337698778478, Tokens per sec: 83929.74441426634, Loss: 2.2862257957458496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7359 , TFLOPS: 102.28481936375798, Tokens per sec: 83579.55894761936, Loss: 2.2606067657470703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7360 , TFLOPS: 100.28652468296809, Tokens per sec: 81946.70092326457, Loss: 2.2675507068634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7361 , TFLOPS: 102.765965712857, Tokens per sec: 83972.71601527708, Loss: 2.2722127437591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7362 , TFLOPS: 101.55109243984965, Tokens per sec: 82980.01179028254, Loss: 2.285543203353882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7363 , TFLOPS: 102.0428265648264, Tokens per sec: 83381.82040216353, Loss: 2.258817672729492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7364 , TFLOPS: 102.8220531959685, Tokens per sec: 84018.54654154772, Loss: 2.2718210220336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7365 , TFLOPS: 102.69464486071773, Tokens per sec: 83914.43791103202, Loss: 2.291473150253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7366 , TFLOPS: 101.56143974786734, Tokens per sec: 82988.46684202719, Loss: 2.267024517059326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7367 , TFLOPS: 102.64727350634857, Tokens per sec: 83875.72955792981, Loss: 2.2779228687286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7368 , TFLOPS: 102.83180273194975, Tokens per sec: 84026.5131383734, Loss: 2.2600820064544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7369 , TFLOPS: 101.30046810112292, Tokens per sec: 82775.22019146448, Loss: 2.276024341583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7370 , TFLOPS: 101.48237528230732, Tokens per sec: 82923.86123191769, Loss: 2.2727091312408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7371 , TFLOPS: 102.32402202027185, Tokens per sec: 83611.59244742301, Loss: 2.276961326599121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7372 , TFLOPS: 102.07164032055982, Tokens per sec: 83405.36486370537, Loss: 2.2754878997802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7373 , TFLOPS: 101.66709106244815, Tokens per sec: 83074.79725087796, Loss: 2.2635955810546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7374 , TFLOPS: 102.04164539269306, Tokens per sec: 83380.8552360073, Loss: 2.2500762939453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7375 , TFLOPS: 102.2002026273134, Tokens per sec: 83510.41643404192, Loss: 2.2671918869018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7376 , TFLOPS: 103.42379703652823, Tokens per sec: 84510.2469239336, Loss: 2.28180193901062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7377 , TFLOPS: 102.3584762689761, Tokens per sec: 83639.7459009705, Loss: 2.289855718612671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7378 , TFLOPS: 103.32860069769598, Tokens per sec: 84432.459544902, Loss: 2.2827980518341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7379 , TFLOPS: 100.16373719475727, Tokens per sec: 81846.36810581636, Loss: 2.2750437259674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7380 , TFLOPS: 103.49413794966456, Tokens per sec: 84567.72429479349, Loss: 2.299745559692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7381 , TFLOPS: 101.27085136773712, Tokens per sec: 82751.01959621241, Loss: 2.2683331966400146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7382 , TFLOPS: 100.78613452331419, Tokens per sec: 82354.94498491281, Loss: 2.284414291381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7383 , TFLOPS: 101.97423325536937, Tokens per sec: 83325.77104325734, Loss: 2.2704145908355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7384 , TFLOPS: 102.04541351108678, Tokens per sec: 83383.93426254681, Loss: 2.277000665664673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7385 , TFLOPS: 101.60296264045483, Tokens per sec: 83022.39626645479, Loss: 2.2736270427703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7386 , TFLOPS: 102.2141468107951, Tokens per sec: 83521.81058532002, Loss: 2.2630672454833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7387 , TFLOPS: 100.88554136832545, Tokens per sec: 82436.17287693135, Loss: 2.294126272201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7388 , TFLOPS: 102.18828100649735, Tokens per sec: 83500.6749707833, Loss: 2.251145124435425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7389 , TFLOPS: 102.26625416660634, Tokens per sec: 83564.38884711606, Loss: 2.2643439769744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7390 , TFLOPS: 100.36213609755967, Tokens per sec: 82008.48495653832, Loss: 2.2705726623535156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7391 , TFLOPS: 102.91151073274418, Tokens per sec: 84091.644597689, Loss: 2.2746963500976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7392 , TFLOPS: 101.1676544854499, Tokens per sec: 82666.69476717144, Loss: 2.2709078788757324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7393 , TFLOPS: 102.68427957601882, Tokens per sec: 83905.96817007837, Loss: 2.256056547164917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7394 , TFLOPS: 100.87722148742891, Tokens per sec: 82429.374488078, Loss: 2.2529289722442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7395 , TFLOPS: 101.0029319569566, Tokens per sec: 82532.09574880447, Loss: 2.269744634628296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7396 , TFLOPS: 102.77525601922417, Tokens per sec: 83980.30736376345, Loss: 2.2751762866973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7397 , TFLOPS: 101.81540363780367, Tokens per sec: 83195.9872741063, Loss: 2.2726454734802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7398 , TFLOPS: 99.88046879672575, Tokens per sec: 81614.90220580752, Loss: 2.2952945232391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7399 , TFLOPS: 102.1720615435457, Tokens per sec: 83487.42163007938, Loss: 2.2710134983062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7400 , TFLOPS: 101.37239607317224, Tokens per sec: 82833.99438901672, Loss: 2.273477077484131 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/model.pt +[rank0]:[2024-08-28 06:53:48,001] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007787453010678291, 'preprocessing_with_comm': 0.0016522840014658868, 'state_converting': 3.4003325770027004, : 3.4114473620138597}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400, took 15.99s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005400 + eval ppl=7.417962074279785, eval loss=2.003904342651367 +------------------------------------------------------------------ +iteration: 7401 , TFLOPS: 101.98889042493646, Tokens per sec: 83337.7477938197, Loss: 2.2817652225494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7402 , TFLOPS: 101.48572196042946, Tokens per sec: 82926.59588876246, Loss: 2.27182936668396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7403 , TFLOPS: 101.39622552854779, Tokens per sec: 82853.46605042882, Loss: 2.2598888874053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7404 , TFLOPS: 102.62043247950785, Tokens per sec: 83853.79706394878, Loss: 2.2813377380371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7405 , TFLOPS: 101.68966196761528, Tokens per sec: 83093.24051851682, Loss: 2.2823781967163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7406 , TFLOPS: 101.73260187936425, Tokens per sec: 83128.32782577856, Loss: 2.248530864715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7407 , TFLOPS: 101.85278489630724, Tokens per sec: 83226.53246270877, Loss: 2.266655921936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7408 , TFLOPS: 102.00022624100997, Tokens per sec: 83347.01058094324, Loss: 2.283783435821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7409 , TFLOPS: 103.37673797260753, Tokens per sec: 84471.79375139592, Loss: 2.2726612091064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7410 , TFLOPS: 101.33736082646105, Tokens per sec: 82805.36618704152, Loss: 2.2735934257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7411 , TFLOPS: 102.21371631968287, Tokens per sec: 83521.4588199504, Loss: 2.282160758972168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7412 , TFLOPS: 102.76203656245494, Tokens per sec: 83969.50540533822, Loss: 2.2674434185028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7413 , TFLOPS: 102.02482566127382, Tokens per sec: 83367.11140048625, Loss: 2.277008295059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7414 , TFLOPS: 102.11168649183112, Tokens per sec: 83438.0876211313, Loss: 2.2676937580108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7415 , TFLOPS: 101.49139724571157, Tokens per sec: 82931.23331046129, Loss: 2.285970687866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7416 , TFLOPS: 102.86089395374945, Tokens per sec: 84050.28432458061, Loss: 2.2839393615722656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7417 , TFLOPS: 100.70501462044622, Tokens per sec: 82288.65982406732, Loss: 2.296522378921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7418 , TFLOPS: 102.54679458166251, Tokens per sec: 83793.6256420112, Loss: 2.283554792404175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7419 , TFLOPS: 102.81652068800814, Tokens per sec: 84014.02578687388, Loss: 2.2704222202301025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7420 , TFLOPS: 101.44423291298068, Tokens per sec: 82892.69412006896, Loss: 2.2847683429718018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7421 , TFLOPS: 101.44882023496055, Tokens per sec: 82896.44253894688, Loss: 2.2671282291412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7422 , TFLOPS: 102.95414075726454, Tokens per sec: 84126.47868811905, Loss: 2.261047601699829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7423 , TFLOPS: 101.43636498125753, Tokens per sec: 82886.26503051938, Loss: 2.2752432823181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7424 , TFLOPS: 102.16604492631048, Tokens per sec: 83482.50529725497, Loss: 2.262490749359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7425 , TFLOPS: 100.69818280336224, Tokens per sec: 82283.07737046138, Loss: 2.268352746963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7426 , TFLOPS: 102.36080610070785, Tokens per sec: 83641.64966645373, Loss: 2.2520933151245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7427 , TFLOPS: 101.45330352003725, Tokens per sec: 82900.10594659309, Loss: 2.2819783687591553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7428 , TFLOPS: 102.41421408941689, Tokens per sec: 83685.2907088718, Loss: 2.2702555656433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7429 , TFLOPS: 101.79637723248268, Tokens per sec: 83180.44030853528, Loss: 2.252300262451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7430 , TFLOPS: 101.66076329823947, Tokens per sec: 83069.6266718519, Loss: 2.2609212398529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7431 , TFLOPS: 102.46912294600266, Tokens per sec: 83730.15814906776, Loss: 2.2752506732940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7432 , TFLOPS: 102.94498425346984, Tokens per sec: 84118.99667315901, Loss: 2.276310920715332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7433 , TFLOPS: 102.03493725064419, Tokens per sec: 83375.37384045623, Loss: 2.263097047805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7434 , TFLOPS: 101.30078893060899, Tokens per sec: 82775.48234949644, Loss: 2.2563443183898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7435 , TFLOPS: 102.86768670458447, Tokens per sec: 84055.83485614887, Loss: 2.266193151473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7436 , TFLOPS: 101.52572359538276, Tokens per sec: 82959.28225441648, Loss: 2.295525074005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7437 , TFLOPS: 102.78239555774633, Tokens per sec: 83986.14126447796, Loss: 2.2828521728515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7438 , TFLOPS: 100.78996583549869, Tokens per sec: 82358.07564872553, Loss: 2.262248992919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7439 , TFLOPS: 102.18247929851512, Tokens per sec: 83495.93424584177, Loss: 2.2764649391174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7440 , TFLOPS: 100.2047082082928, Tokens per sec: 81879.84657566396, Loss: 2.2408320903778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7441 , TFLOPS: 102.66948767413056, Tokens per sec: 83893.88132626802, Loss: 2.299382209777832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7442 , TFLOPS: 101.97311677575097, Tokens per sec: 83324.85873902039, Loss: 2.262700319290161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7443 , TFLOPS: 101.72378769975072, Tokens per sec: 83121.12553271918, Loss: 2.263587474822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7444 , TFLOPS: 101.49708640226396, Tokens per sec: 82935.88206673219, Loss: 2.265454053878784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7445 , TFLOPS: 102.11252615443063, Tokens per sec: 83438.7737310562, Loss: 2.252809524536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7446 , TFLOPS: 102.21205369982377, Tokens per sec: 83520.10025046389, Loss: 2.260244846343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7447 , TFLOPS: 102.91459823142648, Tokens per sec: 84094.1674723416, Loss: 2.263340950012207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7448 , TFLOPS: 102.04434507871017, Tokens per sec: 83383.06121894799, Loss: 2.2698981761932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7449 , TFLOPS: 101.44538606593707, Tokens per sec: 82893.63639103442, Loss: 2.2796638011932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7450 , TFLOPS: 103.32244766160963, Tokens per sec: 84427.4317407226, Loss: 2.269746780395508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7451 , TFLOPS: 102.84102247698671, Tokens per sec: 84034.04682938044, Loss: 2.283276319503784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7452 , TFLOPS: 101.44499176547343, Tokens per sec: 82893.31419797539, Loss: 2.2841172218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7453 , TFLOPS: 102.79197377957084, Tokens per sec: 83993.96787609614, Loss: 2.248382806777954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7454 , TFLOPS: 102.10515076185713, Tokens per sec: 83432.74710792491, Loss: 2.2676026821136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7455 , TFLOPS: 101.30332800998816, Tokens per sec: 82777.55709661888, Loss: 2.2815499305725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7456 , TFLOPS: 103.41204800900444, Tokens per sec: 84500.64649109702, Loss: 2.2679250240325928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7457 , TFLOPS: 101.37231002883124, Tokens per sec: 82833.92407997069, Loss: 2.270724296569824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7458 , TFLOPS: 102.19677444036867, Tokens per sec: 83507.61517424, Loss: 2.2991080284118652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7459 , TFLOPS: 101.5568299668603, Tokens per sec: 82984.70007130002, Loss: 2.2793397903442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7460 , TFLOPS: 102.87856685686286, Tokens per sec: 84064.72530865557, Loss: 2.279874086380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7461 , TFLOPS: 100.57975962984277, Tokens per sec: 82186.31074689493, Loss: 2.2679524421691895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7462 , TFLOPS: 102.21712091553904, Tokens per sec: 83524.24080286658, Loss: 2.255819797515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7463 , TFLOPS: 101.51426232107104, Tokens per sec: 82949.91695213693, Loss: 2.2812600135803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7464 , TFLOPS: 102.24671736799031, Tokens per sec: 83548.42482604497, Loss: 2.2841944694519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7465 , TFLOPS: 100.44148657223185, Tokens per sec: 82073.32427205589, Loss: 2.2841103076934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7466 , TFLOPS: 101.64589196513838, Tokens per sec: 83057.47492275282, Loss: 2.2730329036712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7467 , TFLOPS: 102.43521550444557, Tokens per sec: 83702.45150571626, Loss: 2.278895378112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7468 , TFLOPS: 100.87182538414726, Tokens per sec: 82424.96519317852, Loss: 2.2653796672821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7469 , TFLOPS: 102.85770731401689, Tokens per sec: 84047.68043922374, Loss: 2.264817476272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7470 , TFLOPS: 102.11410555486755, Tokens per sec: 83440.06429981059, Loss: 2.285451650619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7471 , TFLOPS: 102.46431662491605, Tokens per sec: 83726.23078038219, Loss: 2.2861719131469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7472 , TFLOPS: 101.06438710690173, Tokens per sec: 82582.31233382091, Loss: 2.299909830093384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7473 , TFLOPS: 103.32529350035963, Tokens per sec: 84429.75714882363, Loss: 2.2663750648498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7474 , TFLOPS: 100.7834102656833, Tokens per sec: 82352.71892387398, Loss: 2.257739543914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7475 , TFLOPS: 100.1750221969589, Tokens per sec: 81855.58937161711, Loss: 2.296429395675659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7476 , TFLOPS: 100.7878150706094, Tokens per sec: 82356.31820336885, Loss: 2.254927158355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7477 , TFLOPS: 102.16206997828716, Tokens per sec: 83479.25726490087, Loss: 2.2762327194213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7478 , TFLOPS: 100.92961824911401, Tokens per sec: 82472.18923086354, Loss: 2.2845277786254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7479 , TFLOPS: 102.37641741636428, Tokens per sec: 83654.4060743476, Loss: 2.2793128490448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7480 , TFLOPS: 102.71320651215075, Tokens per sec: 83929.60511423735, Loss: 2.2526519298553467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7481 , TFLOPS: 100.46957130103546, Tokens per sec: 82096.27302692656, Loss: 2.2723495960235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7482 , TFLOPS: 102.22010714804124, Tokens per sec: 83526.6809303166, Loss: 2.2906908988952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7483 , TFLOPS: 102.90635035249808, Tokens per sec: 84087.42791814992, Loss: 2.276308298110962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7484 , TFLOPS: 102.78874399098423, Tokens per sec: 83991.32873269987, Loss: 2.246767044067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7485 , TFLOPS: 103.38767798876148, Tokens per sec: 84480.73312021638, Loss: 2.2663443088531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7486 , TFLOPS: 102.10828258475298, Tokens per sec: 83435.3062010334, Loss: 2.2820944786071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7487 , TFLOPS: 102.15655101474212, Tokens per sec: 83474.74757771745, Loss: 2.2660136222839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7488 , TFLOPS: 103.40072624680465, Tokens per sec: 84491.39518775551, Loss: 2.2564239501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7489 , TFLOPS: 102.22648154136319, Tokens per sec: 83531.88961119142, Loss: 2.266648292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7490 , TFLOPS: 102.14339107161777, Tokens per sec: 83463.99424942347, Loss: 2.266077756881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7491 , TFLOPS: 102.65005855619965, Tokens per sec: 83878.00529385671, Loss: 2.292189359664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7492 , TFLOPS: 102.67015531770353, Tokens per sec: 83894.42687501818, Loss: 2.272813081741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7493 , TFLOPS: 100.67468873172555, Tokens per sec: 82263.87975973584, Loss: 2.2871134281158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7494 , TFLOPS: 103.41085091080896, Tokens per sec: 84499.66831134546, Loss: 2.267507791519165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7495 , TFLOPS: 102.82955959734343, Tokens per sec: 84024.68021534315, Loss: 2.2823591232299805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7496 , TFLOPS: 101.46498515035437, Tokens per sec: 82909.65130743712, Loss: 2.263129949569702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7497 , TFLOPS: 101.65437857255888, Tokens per sec: 83064.40954814084, Loss: 2.283517360687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7498 , TFLOPS: 102.1187882207487, Tokens per sec: 83443.89062664463, Loss: 2.2735698223114014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7499 , TFLOPS: 102.0109839709659, Tokens per sec: 83355.80099901883, Loss: 2.2709479331970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7500 , TFLOPS: 101.4737303071924, Tokens per sec: 82916.79720020982, Loss: 2.257312297821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7501 , TFLOPS: 101.36325027613216, Tokens per sec: 82826.5211228215, Loss: 2.263619899749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7502 , TFLOPS: 101.56748246187045, Tokens per sec: 82993.40449919243, Loss: 2.271573305130005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7503 , TFLOPS: 100.69059984872868, Tokens per sec: 82276.88114302768, Loss: 2.2695794105529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7504 , TFLOPS: 101.02191416330636, Tokens per sec: 82547.60659825831, Loss: 2.279478073120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7505 , TFLOPS: 102.35649961394233, Tokens per sec: 83638.13072525879, Loss: 2.2805850505828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7506 , TFLOPS: 100.36906411166714, Tokens per sec: 82014.14601520848, Loss: 2.276597738265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7507 , TFLOPS: 102.8140262474846, Tokens per sec: 84011.98751530952, Loss: 2.2655529975891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7508 , TFLOPS: 101.7096255112818, Tokens per sec: 83109.55324395405, Loss: 2.2889370918273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7509 , TFLOPS: 102.61307747148675, Tokens per sec: 83847.78709755998, Loss: 2.2715742588043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7510 , TFLOPS: 101.62463789368462, Tokens per sec: 83040.10767383949, Loss: 2.2767333984375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7511 , TFLOPS: 102.92841894397417, Tokens per sec: 84105.46073234112, Loss: 2.272261619567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7512 , TFLOPS: 101.48022981479667, Tokens per sec: 82922.10811518549, Loss: 2.2712583541870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7513 , TFLOPS: 102.26294010606509, Tokens per sec: 83561.6808429363, Loss: 2.2756872177124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7514 , TFLOPS: 100.65449684955055, Tokens per sec: 82247.38045302535, Loss: 2.278836250305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7515 , TFLOPS: 102.77839310452296, Tokens per sec: 83982.87075691685, Loss: 2.2595717906951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7516 , TFLOPS: 101.52035167847694, Tokens per sec: 82954.89272282731, Loss: 2.2572503089904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7517 , TFLOPS: 102.07641830913207, Tokens per sec: 83409.26908116412, Loss: 2.2765684127807617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7518 , TFLOPS: 102.60916522946354, Tokens per sec: 83844.59030389274, Loss: 2.261106491088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7519 , TFLOPS: 101.01797633389207, Tokens per sec: 82544.38889647514, Loss: 2.285414457321167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7520 , TFLOPS: 102.19377427918411, Tokens per sec: 83505.16366530507, Loss: 2.2674379348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7521 , TFLOPS: 101.81221861806355, Tokens per sec: 83193.38471248695, Loss: 2.286261558532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7522 , TFLOPS: 102.1217800379659, Tokens per sec: 83446.33531750903, Loss: 2.2743120193481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7523 , TFLOPS: 102.65481011089472, Tokens per sec: 83881.88791151393, Loss: 2.2633047103881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7524 , TFLOPS: 102.93029447109079, Tokens per sec: 84106.99327382847, Loss: 2.284498691558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7525 , TFLOPS: 102.09035363898579, Tokens per sec: 83420.6559979149, Loss: 2.2727906703948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7526 , TFLOPS: 103.45627930396762, Tokens per sec: 84536.78902082628, Loss: 2.270299196243286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7527 , TFLOPS: 102.27717346056991, Tokens per sec: 83573.31128330134, Loss: 2.2678956985473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7528 , TFLOPS: 102.01966973863283, Tokens per sec: 83362.89836337103, Loss: 2.2785801887512207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7529 , TFLOPS: 102.7511350468538, Tokens per sec: 83960.59749631055, Loss: 2.2681100368499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7530 , TFLOPS: 102.20641686600608, Tokens per sec: 83515.49424844644, Loss: 2.2751801013946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7531 , TFLOPS: 100.6669987379126, Tokens per sec: 82257.59606783323, Loss: 2.2742040157318115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7532 , TFLOPS: 102.73793918838072, Tokens per sec: 83949.81482066058, Loss: 2.2596616744995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7533 , TFLOPS: 102.73527730712543, Tokens per sec: 83947.63972896397, Loss: 2.2805707454681396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7534 , TFLOPS: 102.14756135255887, Tokens per sec: 83467.40189333302, Loss: 2.287461996078491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7535 , TFLOPS: 100.45894749590438, Tokens per sec: 82087.59204227288, Loss: 2.2813291549682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7536 , TFLOPS: 103.42638344471257, Tokens per sec: 84512.3603446411, Loss: 2.2891101837158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7537 , TFLOPS: 100.92640903607355, Tokens per sec: 82469.56690027568, Loss: 2.2738771438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7538 , TFLOPS: 101.59804741332644, Tokens per sec: 83018.37990783897, Loss: 2.266592025756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7539 , TFLOPS: 102.03694738870774, Tokens per sec: 83377.01637601352, Loss: 2.2678475379943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7540 , TFLOPS: 101.50587705288261, Tokens per sec: 82943.06513364417, Loss: 2.275545120239258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7541 , TFLOPS: 102.08112046761325, Tokens per sec: 83413.1113359034, Loss: 2.2673959732055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7542 , TFLOPS: 101.12246648095777, Tokens per sec: 82629.7704854582, Loss: 2.3041741847991943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7543 , TFLOPS: 102.69487362075944, Tokens per sec: 83914.6248367509, Loss: 2.247771739959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7544 , TFLOPS: 100.7334527005426, Tokens per sec: 82311.89731137537, Loss: 2.286294937133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7545 , TFLOPS: 102.71808149438853, Tokens per sec: 83933.58858771704, Loss: 2.291166305541992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7546 , TFLOPS: 101.53137960086237, Tokens per sec: 82963.9039220924, Loss: 2.287212610244751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7547 , TFLOPS: 102.04647978683539, Tokens per sec: 83384.80554390939, Loss: 2.269031047821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7548 , TFLOPS: 100.96029255135002, Tokens per sec: 82497.25399284737, Loss: 2.270709753036499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7549 , TFLOPS: 102.23633627823993, Tokens per sec: 83539.94217037692, Loss: 2.2680768966674805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7550 , TFLOPS: 101.48272075380808, Tokens per sec: 82924.1435255742, Loss: 2.263193130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7551 , TFLOPS: 102.35868629367381, Tokens per sec: 83639.91751755752, Loss: 2.248900890350342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7552 , TFLOPS: 100.81157714578231, Tokens per sec: 82375.73480668287, Loss: 2.287574291229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7553 , TFLOPS: 102.84854849364251, Tokens per sec: 84040.19653132682, Loss: 2.2832236289978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7554 , TFLOPS: 101.51696280625858, Tokens per sec: 82952.12358810034, Loss: 2.257307529449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7555 , TFLOPS: 102.77068601025807, Tokens per sec: 83976.57309179475, Loss: 2.274134635925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7556 , TFLOPS: 102.68806486643989, Tokens per sec: 83909.06122832345, Loss: 2.278759717941284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7557 , TFLOPS: 100.99552782084994, Tokens per sec: 82526.04563859227, Loss: 2.275827646255493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7558 , TFLOPS: 102.06942643753183, Tokens per sec: 83403.55584289278, Loss: 2.286759376525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7559 , TFLOPS: 102.15356464671525, Tokens per sec: 83472.30733952674, Loss: 2.286054849624634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7560 , TFLOPS: 102.10771954508384, Tokens per sec: 83434.84612682581, Loss: 2.2780063152313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7561 , TFLOPS: 103.37962074282545, Tokens per sec: 84474.14933715008, Loss: 2.283250093460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7562 , TFLOPS: 102.74855860990976, Tokens per sec: 83958.49222336018, Loss: 2.262336492538452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7563 , TFLOPS: 101.62943471402377, Tokens per sec: 83044.02727921976, Loss: 2.272494316101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7564 , TFLOPS: 102.71367711679693, Tokens per sec: 83929.98965740818, Loss: 2.2779524326324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7565 , TFLOPS: 102.8807398741548, Tokens per sec: 84066.50093702326, Loss: 2.281623601913452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7566 , TFLOPS: 102.14145592479626, Tokens per sec: 83462.4129911405, Loss: 2.234982967376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7567 , TFLOPS: 102.90001521341907, Tokens per sec: 84082.25131292747, Loss: 2.2737295627593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7568 , TFLOPS: 102.22806392888754, Tokens per sec: 83533.18262077185, Loss: 2.243640661239624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7569 , TFLOPS: 101.15796989506072, Tokens per sec: 82658.7812390609, Loss: 2.2635629177093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7570 , TFLOPS: 101.99638924756343, Tokens per sec: 83343.87527482507, Loss: 2.258695602416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7571 , TFLOPS: 102.11111293388133, Tokens per sec: 83437.61895216584, Loss: 2.2814154624938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7572 , TFLOPS: 102.21457714266792, Tokens per sec: 83522.16222057104, Loss: 2.3005056381225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7573 , TFLOPS: 100.98179073737285, Tokens per sec: 82514.82071405915, Loss: 2.245136260986328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7574 , TFLOPS: 102.14424275068517, Tokens per sec: 83464.69017831361, Loss: 2.2754623889923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7575 , TFLOPS: 101.5508589781297, Tokens per sec: 82979.82102270133, Loss: 2.285522937774658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7576 , TFLOPS: 101.04430093246614, Tokens per sec: 82565.89940362547, Loss: 2.265289783477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7577 , TFLOPS: 102.12090212326466, Tokens per sec: 83445.61795080712, Loss: 2.286044120788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7578 , TFLOPS: 101.77846270812599, Tokens per sec: 83165.80188952254, Loss: 2.266704559326172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7579 , TFLOPS: 100.7279605481177, Tokens per sec: 82307.4095322484, Loss: 2.272221326828003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7580 , TFLOPS: 102.43462849066701, Tokens per sec: 83701.97184164672, Loss: 2.2923314571380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7581 , TFLOPS: 101.71028701105348, Tokens per sec: 83110.09377244602, Loss: 2.2676310539245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7582 , TFLOPS: 101.49078956492501, Tokens per sec: 82930.7367588475, Loss: 2.300210952758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7583 , TFLOPS: 102.8213711770956, Tokens per sec: 84017.98924636997, Loss: 2.269183874130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7584 , TFLOPS: 101.52372961384053, Tokens per sec: 82957.65292076867, Loss: 2.267542600631714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7585 , TFLOPS: 101.37979876822457, Tokens per sec: 82840.04332170631, Loss: 2.2799923419952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7586 , TFLOPS: 102.06004182166217, Tokens per sec: 83395.88743168418, Loss: 2.3035943508148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7587 , TFLOPS: 102.14275046346513, Tokens per sec: 83463.47079201124, Loss: 2.290562629699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7588 , TFLOPS: 102.06231405597676, Tokens per sec: 83397.74413283514, Loss: 2.272341012954712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7589 , TFLOPS: 101.86567208136327, Tokens per sec: 83237.06291336371, Loss: 2.275285005569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7590 , TFLOPS: 100.99747161735849, Tokens per sec: 82527.63396475719, Loss: 2.2900702953338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7591 , TFLOPS: 101.59974487309519, Tokens per sec: 83019.76694591252, Loss: 2.258474349975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7592 , TFLOPS: 101.93768981238644, Tokens per sec: 83295.91045528406, Loss: 2.2761223316192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7593 , TFLOPS: 103.34247524882348, Tokens per sec: 84443.796797791, Loss: 2.2710487842559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7594 , TFLOPS: 102.22281898643648, Tokens per sec: 83528.89684327826, Loss: 2.2922585010528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7595 , TFLOPS: 101.35391057119138, Tokens per sec: 82818.889409489, Loss: 2.2894139289855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7596 , TFLOPS: 103.37519237098715, Tokens per sec: 84470.53080052458, Loss: 2.293933153152466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7597 , TFLOPS: 101.8681851521036, Tokens per sec: 83239.11640815783, Loss: 2.2527666091918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7598 , TFLOPS: 101.37676545081062, Tokens per sec: 82837.56471996283, Loss: 2.285869836807251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7599 , TFLOPS: 103.36155281062611, Tokens per sec: 84459.38556464005, Loss: 2.26584529876709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7600 , TFLOPS: 101.86533817654477, Tokens per sec: 83236.79007114103, Loss: 2.282127618789673 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/model.pt +[rank0]:[2024-08-28 07:46:37,235] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007776242011459544, 'preprocessing_with_comm': 0.0017806180112529546, 'state_converting': 3.3171029789955355, : 3.3283406410191674}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600, took 15.75s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005600 + eval ppl=8.140573501586914, eval loss=2.096860647201538 +------------------------------------------------------------------ +iteration: 7601 , TFLOPS: 100.75895330467254, Tokens per sec: 82332.73451145392, Loss: 2.2647621631622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7602 , TFLOPS: 100.1994238425791, Tokens per sec: 81875.52858440761, Loss: 2.25238037109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7603 , TFLOPS: 102.36128768142854, Tokens per sec: 83642.04317845759, Loss: 2.276352882385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7604 , TFLOPS: 100.94137766737533, Tokens per sec: 82481.79815423921, Loss: 2.2799291610717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7605 , TFLOPS: 101.41077363179829, Tokens per sec: 82865.35368009632, Loss: 2.274852752685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7606 , TFLOPS: 103.34260360756979, Tokens per sec: 84443.90168302685, Loss: 2.274928092956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7607 , TFLOPS: 101.8753972458279, Tokens per sec: 83245.00959559577, Loss: 2.271538734436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7608 , TFLOPS: 101.96430917019066, Tokens per sec: 83317.66181778912, Loss: 2.289520025253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7609 , TFLOPS: 102.70944085922163, Tokens per sec: 83926.52810229243, Loss: 2.284867525100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7610 , TFLOPS: 102.79494666025901, Tokens per sec: 83996.3970934352, Loss: 2.2816529273986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7611 , TFLOPS: 103.43398009117207, Tokens per sec: 84518.56776001824, Loss: 2.2700254917144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7612 , TFLOPS: 101.39786237867169, Tokens per sec: 82854.80356280135, Loss: 2.26766037940979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7613 , TFLOPS: 102.49943766288702, Tokens per sec: 83754.92908460421, Loss: 2.272897720336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7614 , TFLOPS: 102.63913677248301, Tokens per sec: 83869.0808232301, Loss: 2.2844808101654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7615 , TFLOPS: 102.6669303762851, Tokens per sec: 83891.7916923678, Loss: 2.263897180557251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7616 , TFLOPS: 103.41357299946553, Tokens per sec: 84501.89260005954, Loss: 2.28300404548645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7617 , TFLOPS: 102.02871421068416, Tokens per sec: 83370.28883430958, Loss: 2.3038625717163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7618 , TFLOPS: 101.99834214350804, Tokens per sec: 83345.47103637342, Loss: 2.28291392326355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7619 , TFLOPS: 102.06864523727401, Tokens per sec: 83402.91750405224, Loss: 2.262373924255371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7620 , TFLOPS: 103.25707058568396, Tokens per sec: 84374.01044902808, Loss: 2.2662148475646973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7621 , TFLOPS: 102.49696226114524, Tokens per sec: 83752.9063701185, Loss: 2.2569580078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7622 , TFLOPS: 102.13196752454391, Tokens per sec: 83454.65977504116, Loss: 2.2827019691467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7623 , TFLOPS: 101.38806066780555, Tokens per sec: 82846.79433253413, Loss: 2.2962777614593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7624 , TFLOPS: 102.85363607188297, Tokens per sec: 84044.353722473, Loss: 2.2773125171661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7625 , TFLOPS: 102.76202444775893, Tokens per sec: 83969.49550610824, Loss: 2.265350103378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7626 , TFLOPS: 101.75461141216623, Tokens per sec: 83146.31238160683, Loss: 2.274705171585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7627 , TFLOPS: 101.44324963512322, Tokens per sec: 82891.89065841981, Loss: 2.263641595840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7628 , TFLOPS: 102.03541119854474, Tokens per sec: 83375.76111548627, Loss: 2.2675766944885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7629 , TFLOPS: 102.15813439698978, Tokens per sec: 83476.0414001119, Loss: 2.2709240913391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7630 , TFLOPS: 100.86943991077678, Tokens per sec: 82423.01596147993, Loss: 2.2807273864746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7631 , TFLOPS: 102.84029393993605, Tokens per sec: 84033.45152300215, Loss: 2.255554676055908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7632 , TFLOPS: 101.41298013338798, Tokens per sec: 82867.15666934564, Loss: 2.2888107299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7633 , TFLOPS: 100.94829845185374, Tokens per sec: 82487.45330539337, Loss: 2.2524795532226562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7634 , TFLOPS: 101.89874765136635, Tokens per sec: 83264.08981305415, Loss: 2.2773468494415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7635 , TFLOPS: 101.15328979828658, Tokens per sec: 82654.95701151036, Loss: 2.2581639289855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7636 , TFLOPS: 100.37720705501707, Tokens per sec: 82020.79982383772, Loss: 2.276580333709717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7637 , TFLOPS: 101.6956028825673, Tokens per sec: 83098.09499305687, Loss: 2.2598440647125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7638 , TFLOPS: 101.51933633263994, Tokens per sec: 82954.06305760656, Loss: 2.280036211013794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7639 , TFLOPS: 100.26429995263568, Tokens per sec: 81928.5405239945, Loss: 2.27670955657959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7640 , TFLOPS: 101.17472124182815, Tokens per sec: 82672.4691957226, Loss: 2.2835869789123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7641 , TFLOPS: 102.38636765250631, Tokens per sec: 83662.53666844113, Loss: 2.2791495323181152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7642 , TFLOPS: 100.30629887545497, Tokens per sec: 81962.8589249786, Loss: 2.267545223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7643 , TFLOPS: 102.2075022472946, Tokens per sec: 83516.3811414375, Loss: 2.247371196746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7644 , TFLOPS: 102.76582395713297, Tokens per sec: 83972.60018302583, Loss: 2.2630739212036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7645 , TFLOPS: 102.80386098883339, Tokens per sec: 84003.68122078857, Loss: 2.2674753665924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7646 , TFLOPS: 102.03075154213299, Tokens per sec: 83371.953590282, Loss: 2.2627112865448 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7647 , TFLOPS: 102.69947429943707, Tokens per sec: 83918.38416973039, Loss: 2.2734761238098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7648 , TFLOPS: 102.6131463944071, Tokens per sec: 83847.84341625242, Loss: 2.294882297515869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7649 , TFLOPS: 103.39421671311153, Tokens per sec: 84486.07608020456, Loss: 2.2785563468933105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7650 , TFLOPS: 101.57031080753028, Tokens per sec: 82995.71561323917, Loss: 2.2658705711364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7651 , TFLOPS: 102.5611570333376, Tokens per sec: 83805.36157099732, Loss: 2.2981104850769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7652 , TFLOPS: 102.087967396237, Tokens per sec: 83418.70613753752, Loss: 2.2751593589782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7653 , TFLOPS: 102.02027548492208, Tokens per sec: 83363.39333425727, Loss: 2.287125587463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7654 , TFLOPS: 103.46019160531011, Tokens per sec: 84539.98586296482, Loss: 2.25400710105896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7655 , TFLOPS: 102.11465514367362, Tokens per sec: 83440.51338297171, Loss: 2.2785556316375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7656 , TFLOPS: 102.11075876812922, Tokens per sec: 83437.32955421275, Loss: 2.273782730102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7657 , TFLOPS: 102.19753416611567, Tokens per sec: 83508.23596570495, Loss: 2.270629644393921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7658 , TFLOPS: 103.3139855333902, Tokens per sec: 84420.51711791994, Loss: 2.2653424739837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7659 , TFLOPS: 103.34150013684534, Tokens per sec: 84443.00000869205, Loss: 2.252209186553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7660 , TFLOPS: 102.00090724650417, Tokens per sec: 83347.5670480632, Loss: 2.2622392177581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7661 , TFLOPS: 101.49785572757646, Tokens per sec: 82936.51070224923, Loss: 2.282026529312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7662 , TFLOPS: 101.98615329509131, Tokens per sec: 83335.51121456323, Loss: 2.2710397243499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7663 , TFLOPS: 103.03195109456759, Tokens per sec: 84190.05951774566, Loss: 2.287271738052368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7664 , TFLOPS: 101.46573152152915, Tokens per sec: 82910.26118653733, Loss: 2.290318489074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7665 , TFLOPS: 101.69617806215709, Tokens per sec: 83098.56498710612, Loss: 2.2611494064331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7666 , TFLOPS: 102.08962141819296, Tokens per sec: 83420.05768145603, Loss: 2.2562875747680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7667 , TFLOPS: 101.67128977276813, Tokens per sec: 83078.22812516484, Loss: 2.28899884223938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7668 , TFLOPS: 100.90332110724245, Tokens per sec: 82450.70115929162, Loss: 2.259376287460327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7669 , TFLOPS: 102.1426157177501, Tokens per sec: 83463.36068781881, Loss: 2.2674739360809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7670 , TFLOPS: 102.1267181600408, Tokens per sec: 83450.370384175, Loss: 2.2705776691436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7671 , TFLOPS: 100.94953496457782, Tokens per sec: 82488.46369177088, Loss: 2.278367280960083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7672 , TFLOPS: 102.11979708859873, Tokens per sec: 83444.71499853545, Loss: 2.2683396339416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7673 , TFLOPS: 100.67785772765006, Tokens per sec: 82266.46922788296, Loss: 2.285951852798462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7674 , TFLOPS: 101.05833953291524, Tokens per sec: 82577.37070543815, Loss: 2.2584424018859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7675 , TFLOPS: 101.6443556875044, Tokens per sec: 83056.2195907508, Loss: 2.2887954711914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7676 , TFLOPS: 101.54532291781402, Tokens per sec: 82975.29736530651, Loss: 2.2756402492523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7677 , TFLOPS: 100.29259674752466, Tokens per sec: 81951.66256352258, Loss: 2.2827651500701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7678 , TFLOPS: 101.22521373664931, Tokens per sec: 82713.72791303342, Loss: 2.252692937850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7679 , TFLOPS: 102.41468141601216, Tokens per sec: 83685.67257346287, Loss: 2.261347770690918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7680 , TFLOPS: 100.31255040458548, Tokens per sec: 81967.96721035983, Loss: 2.2701058387756348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7681 , TFLOPS: 102.29200234890543, Tokens per sec: 83585.42834968983, Loss: 2.2483816146850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7682 , TFLOPS: 103.39676251767251, Tokens per sec: 84488.15632264617, Loss: 2.28468656539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7683 , TFLOPS: 102.04242449991102, Tokens per sec: 83381.4918645711, Loss: 2.2619056701660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7684 , TFLOPS: 102.75403490142419, Tokens per sec: 83962.9670421289, Loss: 2.2692580223083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7685 , TFLOPS: 102.05053999494109, Tokens per sec: 83388.1232444716, Loss: 2.2767210006713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7686 , TFLOPS: 103.4314807129142, Tokens per sec: 84516.525453704, Loss: 2.2649590969085693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7687 , TFLOPS: 103.3942368139756, Tokens per sec: 84486.09250513802, Loss: 2.2628328800201416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7688 , TFLOPS: 101.20115907333906, Tokens per sec: 82694.0722284205, Loss: 2.2637834548950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7689 , TFLOPS: 102.65846624182524, Tokens per sec: 83884.87543021445, Loss: 2.280604362487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7690 , TFLOPS: 101.89019385292924, Tokens per sec: 83257.10028415683, Loss: 2.2787442207336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7691 , TFLOPS: 101.99854710991859, Tokens per sec: 83345.63851970379, Loss: 2.244425058364868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7692 , TFLOPS: 103.41283599501023, Tokens per sec: 84501.29037474692, Loss: 2.2775352001190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7693 , TFLOPS: 101.32362821358126, Tokens per sec: 82794.14491554853, Loss: 2.2655060291290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7694 , TFLOPS: 102.13490761954148, Tokens per sec: 83457.06220234846, Loss: 2.267165422439575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7695 , TFLOPS: 101.62334860756353, Tokens per sec: 83039.05416496076, Loss: 2.2768099308013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7696 , TFLOPS: 103.32783148983802, Tokens per sec: 84431.83100536009, Loss: 2.2537310123443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7697 , TFLOPS: 103.38351475362997, Tokens per sec: 84477.33123361922, Loss: 2.249703884124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7698 , TFLOPS: 102.09410773729653, Tokens per sec: 83423.72356828369, Loss: 2.2830698490142822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7699 , TFLOPS: 101.43529237891678, Tokens per sec: 82885.38857953553, Loss: 2.282716989517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7700 , TFLOPS: 102.69850230448364, Tokens per sec: 83917.58992763255, Loss: 2.2787179946899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7701 , TFLOPS: 102.85664201273886, Tokens per sec: 84046.80995412615, Loss: 2.261075019836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7702 , TFLOPS: 101.69505703509193, Tokens per sec: 83097.64896703338, Loss: 2.2669754028320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7703 , TFLOPS: 102.09539936881924, Tokens per sec: 83424.77899365046, Loss: 2.277859687805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7704 , TFLOPS: 100.79676710265458, Tokens per sec: 82363.63313919879, Loss: 2.2721049785614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7705 , TFLOPS: 102.17967636643041, Tokens per sec: 83493.64389788145, Loss: 2.2938907146453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7706 , TFLOPS: 100.89490925359539, Tokens per sec: 82443.82761713624, Loss: 2.2895755767822266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7707 , TFLOPS: 102.85492526996838, Tokens per sec: 84045.4071594155, Loss: 2.275928258895874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7708 , TFLOPS: 102.28007968284001, Tokens per sec: 83575.68603233104, Loss: 2.2736449241638184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7709 , TFLOPS: 100.88472682171353, Tokens per sec: 82435.50728992646, Loss: 2.2885475158691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7710 , TFLOPS: 101.51880693993053, Tokens per sec: 82953.63047719591, Loss: 2.2754902839660645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7711 , TFLOPS: 101.42530442148994, Tokens per sec: 82877.22716240917, Loss: 2.238478183746338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7712 , TFLOPS: 100.94778365492948, Tokens per sec: 82487.03265157451, Loss: 2.2960205078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7713 , TFLOPS: 101.60307195270177, Tokens per sec: 83022.4855883057, Loss: 2.2872159481048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7714 , TFLOPS: 101.54385125177943, Tokens per sec: 82974.0948291056, Loss: 2.2787160873413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7715 , TFLOPS: 100.30172281489021, Tokens per sec: 81959.11970809285, Loss: 2.272597551345825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7716 , TFLOPS: 101.10397225416546, Tokens per sec: 82614.65837666245, Loss: 2.2703733444213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7717 , TFLOPS: 102.35661762038349, Tokens per sec: 83638.22715135962, Loss: 2.289562702178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7718 , TFLOPS: 100.8660898244576, Tokens per sec: 82420.27851970965, Loss: 2.2457265853881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7719 , TFLOPS: 101.42009248694322, Tokens per sec: 82872.96835652391, Loss: 2.289639711380005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7720 , TFLOPS: 103.31605842300323, Tokens per sec: 84422.21092938367, Loss: 2.274543285369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7721 , TFLOPS: 102.83162135878253, Tokens per sec: 84026.3649336895, Loss: 2.254283905029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7722 , TFLOPS: 99.99174714263026, Tokens per sec: 81705.83060680557, Loss: 2.263174533843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7723 , TFLOPS: 102.86410251149866, Tokens per sec: 84052.90611971311, Loss: 2.2703964710235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7724 , TFLOPS: 103.34505834948284, Tokens per sec: 84445.90751583468, Loss: 2.283491373062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7725 , TFLOPS: 102.07163210561559, Tokens per sec: 83405.35815106297, Loss: 2.287592649459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7726 , TFLOPS: 102.73685529122635, Tokens per sec: 83948.92914039374, Loss: 2.2619948387145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7727 , TFLOPS: 101.95889577848322, Tokens per sec: 83313.23839607189, Loss: 2.269869565963745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7728 , TFLOPS: 102.72362444367099, Tokens per sec: 83938.11787426395, Loss: 2.283447742462158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7729 , TFLOPS: 101.57601595396328, Tokens per sec: 83000.37743525312, Loss: 2.2779245376586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7730 , TFLOPS: 103.38195647514934, Tokens per sec: 84476.05792415903, Loss: 2.270550012588501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7731 , TFLOPS: 101.24964298642234, Tokens per sec: 82733.6896818878, Loss: 2.2669732570648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7732 , TFLOPS: 102.05229498374847, Tokens per sec: 83389.55729100354, Loss: 2.266946315765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7733 , TFLOPS: 101.05766538183569, Tokens per sec: 82576.81983923685, Loss: 2.2882673740386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7734 , TFLOPS: 103.29178126855003, Tokens per sec: 84402.37344153042, Loss: 2.2754878997802734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7735 , TFLOPS: 102.18910068776785, Tokens per sec: 83501.3447534499, Loss: 2.2593979835510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7736 , TFLOPS: 101.52172499724462, Tokens per sec: 82956.01489694473, Loss: 2.264799118041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7737 , TFLOPS: 101.5165422560317, Tokens per sec: 82951.77994568403, Loss: 2.2519218921661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7738 , TFLOPS: 102.31268129401404, Tokens per sec: 83602.32564806147, Loss: 2.2672119140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7739 , TFLOPS: 102.62939430659702, Tokens per sec: 83861.12000356127, Loss: 2.294719934463501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7740 , TFLOPS: 101.33615429422133, Tokens per sec: 82804.38029848953, Loss: 2.2861969470977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7741 , TFLOPS: 100.95155646405932, Tokens per sec: 82490.11551103526, Loss: 2.2798638343811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7742 , TFLOPS: 102.66252768634844, Tokens per sec: 83888.19414108523, Loss: 2.287090301513672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7743 , TFLOPS: 101.57038813513746, Tokens per sec: 82995.77879961759, Loss: 2.2895922660827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7744 , TFLOPS: 100.1057137637804, Tokens per sec: 81798.95566670917, Loss: 2.256901979446411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7745 , TFLOPS: 101.91695963324901, Tokens per sec: 83278.97129226857, Loss: 2.2535672187805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7746 , TFLOPS: 101.10202315273077, Tokens per sec: 82613.06571570583, Loss: 2.295375108718872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7747 , TFLOPS: 101.52392194394221, Tokens per sec: 82957.81007864578, Loss: 2.27162504196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7748 , TFLOPS: 101.02492852621286, Tokens per sec: 82550.0697117858, Loss: 2.2707533836364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7749 , TFLOPS: 101.65212622252261, Tokens per sec: 83062.56909494556, Loss: 2.2811191082000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7750 , TFLOPS: 100.98642327681968, Tokens per sec: 82518.60608129334, Loss: 2.2687020301818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7751 , TFLOPS: 101.64365811092289, Tokens per sec: 83055.64958296884, Loss: 2.2574682235717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7752 , TFLOPS: 100.80021802582051, Tokens per sec: 82366.45297735234, Loss: 2.288437604904175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7753 , TFLOPS: 100.57618106312086, Tokens per sec: 82183.38660790595, Loss: 2.284684419631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7754 , TFLOPS: 101.621468823781, Tokens per sec: 83037.51814524214, Loss: 2.2591211795806885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7755 , TFLOPS: 102.39124129575184, Tokens per sec: 83666.5190477958, Loss: 2.280151605606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7756 , TFLOPS: 100.46758402002702, Tokens per sec: 82094.6491684574, Loss: 2.282426357269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7757 , TFLOPS: 101.70383439920874, Tokens per sec: 83104.82117720267, Loss: 2.2774620056152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7758 , TFLOPS: 103.23561414822731, Tokens per sec: 84356.47784164442, Loss: 2.2690701484680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7759 , TFLOPS: 102.8622504303217, Tokens per sec: 84051.3927365066, Loss: 2.2793638706207275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7760 , TFLOPS: 102.08297561462123, Tokens per sec: 83414.62722427938, Loss: 2.264246940612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7761 , TFLOPS: 102.17677299259526, Tokens per sec: 83491.27147638127, Loss: 2.2636263370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7762 , TFLOPS: 103.36616972523237, Tokens per sec: 84463.1581644148, Loss: 2.2809083461761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7763 , TFLOPS: 101.93922150183755, Tokens per sec: 83297.16203816376, Loss: 2.26259708404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7764 , TFLOPS: 102.65987408923182, Tokens per sec: 83886.02581857158, Loss: 2.2704272270202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7765 , TFLOPS: 102.08489715519856, Tokens per sec: 83416.19736454997, Loss: 2.2742536067962646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7766 , TFLOPS: 101.84305785343483, Tokens per sec: 83218.58424558135, Loss: 2.272911787033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7767 , TFLOPS: 102.63826136304982, Tokens per sec: 83868.3655036472, Loss: 2.2764763832092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7768 , TFLOPS: 102.11342957673664, Tokens per sec: 83439.51194067861, Loss: 2.2963263988494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7769 , TFLOPS: 101.97139080167229, Tokens per sec: 83323.44840117017, Loss: 2.239901065826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7770 , TFLOPS: 101.53308352680541, Tokens per sec: 82965.29624384342, Loss: 2.2755515575408936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7771 , TFLOPS: 101.42839036511744, Tokens per sec: 82879.74876638644, Loss: 2.2728211879730225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7772 , TFLOPS: 102.9181331295185, Tokens per sec: 84097.05592856895, Loss: 2.25765323638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7773 , TFLOPS: 102.17200484972419, Tokens per sec: 83487.3753040984, Loss: 2.240302562713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7774 , TFLOPS: 101.4601337781767, Tokens per sec: 82905.6871263453, Loss: 2.273256301879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7775 , TFLOPS: 101.5110119453686, Tokens per sec: 82947.26098647836, Loss: 2.2573447227478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7776 , TFLOPS: 101.71672495087118, Tokens per sec: 83115.35437879838, Loss: 2.2871460914611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7777 , TFLOPS: 102.72469248568396, Tokens per sec: 83938.9905988866, Loss: 2.2912070751190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7778 , TFLOPS: 100.42991939691257, Tokens per sec: 82063.87243533693, Loss: 2.2772011756896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7779 , TFLOPS: 101.22719815362605, Tokens per sec: 82715.34943122855, Loss: 2.28262996673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7780 , TFLOPS: 102.81926057035544, Tokens per sec: 84016.26461527073, Loss: 2.273430347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7781 , TFLOPS: 101.51899466202292, Tokens per sec: 82953.78386975, Loss: 2.27881121635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7782 , TFLOPS: 100.96754098213377, Tokens per sec: 82503.17687223197, Loss: 2.2792820930480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7783 , TFLOPS: 102.82198337333391, Tokens per sec: 84018.48948767562, Loss: 2.2564918994903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7784 , TFLOPS: 101.57956238008879, Tokens per sec: 83003.27531133339, Loss: 2.284125328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7785 , TFLOPS: 101.62602606361787, Tokens per sec: 83041.24198322665, Loss: 2.243725299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7786 , TFLOPS: 101.6382182921659, Tokens per sec: 83051.20456702911, Loss: 2.281651020050049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7787 , TFLOPS: 101.30836105784849, Tokens per sec: 82781.66972958774, Loss: 2.2510368824005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7788 , TFLOPS: 100.96429916866846, Tokens per sec: 82500.52790299777, Loss: 2.2693076133728027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7789 , TFLOPS: 101.1173229587845, Tokens per sec: 82625.56758108438, Loss: 2.2599101066589355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7790 , TFLOPS: 101.57198626494433, Tokens per sec: 82997.08467262231, Loss: 2.2651546001434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7791 , TFLOPS: 100.63206186318675, Tokens per sec: 82229.04824813957, Loss: 2.262530565261841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7792 , TFLOPS: 101.43968691913274, Tokens per sec: 82888.97947147151, Loss: 2.265329122543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7793 , TFLOPS: 101.58832539069273, Tokens per sec: 83010.43579287785, Loss: 2.2936882972717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7794 , TFLOPS: 101.12489268038495, Tokens per sec: 82631.75299546607, Loss: 2.27905535697937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7795 , TFLOPS: 101.69364220709248, Tokens per sec: 83096.49287465411, Loss: 2.2697596549987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7796 , TFLOPS: 103.34068891438223, Tokens per sec: 84442.33713793471, Loss: 2.2842695713043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7797 , TFLOPS: 102.89356910333, Tokens per sec: 84076.98403043606, Loss: 2.2614529132843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7798 , TFLOPS: 102.08743831364713, Tokens per sec: 83418.27381053352, Loss: 2.280055284500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7799 , TFLOPS: 102.07965993562715, Tokens per sec: 83411.91789762014, Loss: 2.287020206451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7800 , TFLOPS: 103.2351058396812, Tokens per sec: 84356.06248964637, Loss: 2.2801878452301025 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/optimizer.pt +[rank0]:[2024-08-28 08:39:28,140] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007675108005059883, 'preprocessing_with_comm': 0.0015208319819066674, 'state_converting': 3.294349855015753, : 3.3051855779776815}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800, took 15.62s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0005800 + eval ppl=7.772495269775391, eval loss=2.050591230392456 +------------------------------------------------------------------ +iteration: 7801 , TFLOPS: 99.51182910400368, Tokens per sec: 81313.67722325442, Loss: 2.260206699371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7802 , TFLOPS: 101.64424505097958, Tokens per sec: 83056.12918679834, Loss: 2.273620367050171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7803 , TFLOPS: 100.28176760226116, Tokens per sec: 81942.81379016077, Loss: 2.2798962593078613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7804 , TFLOPS: 99.70811747993667, Tokens per sec: 81474.06950814158, Loss: 2.2457051277160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7805 , TFLOPS: 101.11258587042666, Tokens per sec: 82621.6967842437, Loss: 2.2642805576324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7806 , TFLOPS: 100.9289176645065, Tokens per sec: 82471.61676514609, Loss: 2.2957613468170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7807 , TFLOPS: 102.28476906072002, Tokens per sec: 83579.51784371212, Loss: 2.275714874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7808 , TFLOPS: 102.25373448191023, Tokens per sec: 83554.15869046553, Loss: 2.2718002796173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7809 , TFLOPS: 103.39991949437359, Tokens per sec: 84490.73596957642, Loss: 2.2509572505950928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7810 , TFLOPS: 103.32885592355257, Tokens per sec: 84432.66809651951, Loss: 2.275059461593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7811 , TFLOPS: 102.75259451145186, Tokens per sec: 83961.79006240358, Loss: 2.2708168029785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7812 , TFLOPS: 103.2991566190665, Tokens per sec: 84408.40003029583, Loss: 2.2764534950256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7813 , TFLOPS: 103.18387355597277, Tokens per sec: 84314.19927179188, Loss: 2.273602247238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7814 , TFLOPS: 100.67405643985408, Tokens per sec: 82263.36309777103, Loss: 2.277461051940918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7815 , TFLOPS: 102.73308032519274, Tokens per sec: 83945.84451847141, Loss: 2.2885665893554688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7816 , TFLOPS: 102.83769415906981, Tokens per sec: 84031.32717513203, Loss: 2.277479648590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7817 , TFLOPS: 101.81269609649311, Tokens per sec: 83193.77487240313, Loss: 2.268676996231079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7818 , TFLOPS: 102.10558242336045, Tokens per sec: 83433.09982965123, Loss: 2.286132335662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7819 , TFLOPS: 101.90367064267187, Tokens per sec: 83268.11251597895, Loss: 2.255484104156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7820 , TFLOPS: 103.41424711178514, Tokens per sec: 84502.44343458909, Loss: 2.2654240131378174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7821 , TFLOPS: 101.01372091415273, Tokens per sec: 82540.91168346188, Loss: 2.2626399993896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7822 , TFLOPS: 102.71432918388514, Tokens per sec: 83930.52247821166, Loss: 2.2580487728118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7823 , TFLOPS: 102.01366870888232, Tokens per sec: 83357.99476748164, Loss: 2.2525041103363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7824 , TFLOPS: 101.58511135243356, Tokens per sec: 83007.80951947956, Loss: 2.2758495807647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7825 , TFLOPS: 102.29353199620297, Tokens per sec: 83586.67826387327, Loss: 2.2577569484710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7826 , TFLOPS: 102.01519363396076, Tokens per sec: 83359.24082301829, Loss: 2.2708020210266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7827 , TFLOPS: 100.78459320799504, Tokens per sec: 82353.68553648856, Loss: 2.2583727836608887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7828 , TFLOPS: 102.2503264958687, Tokens per sec: 83551.37393734162, Loss: 2.246687889099121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7829 , TFLOPS: 101.63226618949187, Tokens per sec: 83046.3409507134, Loss: 2.2742817401885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7830 , TFLOPS: 100.89861139676314, Tokens per sec: 82446.85273362027, Loss: 2.2569146156311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7831 , TFLOPS: 100.96075950134077, Tokens per sec: 82497.63554970517, Loss: 2.282104969024658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7832 , TFLOPS: 102.8720861026097, Tokens per sec: 84059.42971752585, Loss: 2.281689405441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7833 , TFLOPS: 100.40268169069563, Tokens per sec: 82041.61580442612, Loss: 2.2844653129577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7834 , TFLOPS: 100.8337085876824, Tokens per sec: 82393.81897757344, Loss: 2.257322311401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7835 , TFLOPS: 101.56329408100287, Tokens per sec: 82989.98206536702, Loss: 2.2769739627838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7836 , TFLOPS: 99.62980651846541, Tokens per sec: 81410.07960561993, Loss: 2.2576608657836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7837 , TFLOPS: 101.21915911347432, Tokens per sec: 82708.78052457728, Loss: 2.2958226203918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7838 , TFLOPS: 101.14409568102559, Tokens per sec: 82647.44426161876, Loss: 2.296938896179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7839 , TFLOPS: 101.15991092481747, Tokens per sec: 82660.36730444171, Loss: 2.2828657627105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7840 , TFLOPS: 101.72935596633532, Tokens per sec: 83125.6755066854, Loss: 2.2761569023132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7841 , TFLOPS: 101.6607048467589, Tokens per sec: 83069.57890964246, Loss: 2.2515063285827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7842 , TFLOPS: 101.63992313594323, Tokens per sec: 83052.59763876615, Loss: 2.280123233795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7843 , TFLOPS: 101.26189649746831, Tokens per sec: 82743.7023411969, Loss: 2.271731376647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7844 , TFLOPS: 102.74243732020906, Tokens per sec: 83953.49035996973, Loss: 2.270723342895508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7845 , TFLOPS: 101.72139893895186, Tokens per sec: 83119.17361477826, Loss: 2.239131212234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7846 , TFLOPS: 101.6556959476238, Tokens per sec: 83065.48600921863, Loss: 2.2329137325286865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7847 , TFLOPS: 103.35779598298895, Tokens per sec: 84456.31576406838, Loss: 2.2836551666259766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7848 , TFLOPS: 103.40801791251002, Tokens per sec: 84497.35339550754, Loss: 2.2697384357452393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7849 , TFLOPS: 102.79678841455124, Tokens per sec: 83997.90203827836, Loss: 2.2564525604248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7850 , TFLOPS: 101.4235398776923, Tokens per sec: 82875.78530824871, Loss: 2.2751216888427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7851 , TFLOPS: 103.37213034975657, Tokens per sec: 84468.02874415363, Loss: 2.269056797027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7852 , TFLOPS: 102.05827362454916, Tokens per sec: 83394.4425923057, Loss: 2.2697527408599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7853 , TFLOPS: 102.1740912817809, Tokens per sec: 83489.08018144149, Loss: 2.2560250759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7854 , TFLOPS: 102.76081418333274, Tokens per sec: 83968.50656789064, Loss: 2.2701807022094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7855 , TFLOPS: 102.1030338819301, Tokens per sec: 83431.01735084319, Loss: 2.2599737644195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7856 , TFLOPS: 100.71693246409717, Tokens per sec: 82298.3982009071, Loss: 2.291769504547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7857 , TFLOPS: 102.57582306208558, Tokens per sec: 83817.34556062447, Loss: 2.2791895866394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7858 , TFLOPS: 103.2803856673114, Tokens per sec: 84393.06180240946, Loss: 2.271221876144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7859 , TFLOPS: 101.0607415503279, Tokens per sec: 82579.33345569947, Loss: 2.291234254837036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7860 , TFLOPS: 102.25140325726156, Tokens per sec: 83552.25378679403, Loss: 2.252815008163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7861 , TFLOPS: 100.93282462085783, Tokens per sec: 82474.80923975476, Loss: 2.267136335372925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7862 , TFLOPS: 102.71660800182866, Tokens per sec: 83932.38455901529, Loss: 2.281778335571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7863 , TFLOPS: 100.89510448990966, Tokens per sec: 82443.98714975447, Loss: 2.2618067264556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7864 , TFLOPS: 102.12729110722161, Tokens per sec: 83450.83855406543, Loss: 2.2675070762634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7865 , TFLOPS: 101.04752756128117, Tokens per sec: 82568.53596509104, Loss: 2.254514455795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7866 , TFLOPS: 100.61871416700691, Tokens per sec: 82218.14150199076, Loss: 2.2534902095794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7867 , TFLOPS: 102.85067415831963, Tokens per sec: 84041.93346665397, Loss: 2.2804436683654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7868 , TFLOPS: 99.29553033809084, Tokens per sec: 81136.93393360145, Loss: 2.262681722640991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7869 , TFLOPS: 101.57164391875641, Tokens per sec: 82996.80493273938, Loss: 2.294363021850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7870 , TFLOPS: 101.49359721704472, Tokens per sec: 82933.03096366992, Loss: 2.2454447746276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7871 , TFLOPS: 101.66328654515108, Tokens per sec: 83071.68848185753, Loss: 2.24574613571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7872 , TFLOPS: 101.51189027069185, Tokens per sec: 82947.97868871009, Loss: 2.270616292953491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7873 , TFLOPS: 101.53800131202792, Tokens per sec: 82969.31469274379, Loss: 2.2766122817993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7874 , TFLOPS: 101.56827958251148, Tokens per sec: 82994.05584698793, Loss: 2.2795236110687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7875 , TFLOPS: 101.17844936091204, Tokens per sec: 82675.51553779638, Loss: 2.2505855560302734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7876 , TFLOPS: 101.7312757875487, Tokens per sec: 83127.24424201943, Loss: 2.273402452468872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7877 , TFLOPS: 101.63526041491134, Tokens per sec: 83048.78760938188, Loss: 2.2702267169952393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7878 , TFLOPS: 101.370931450323, Tokens per sec: 82832.79760798384, Loss: 2.260317325592041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7879 , TFLOPS: 101.54442860141232, Tokens per sec: 82974.56659635293, Loss: 2.2751898765563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7880 , TFLOPS: 102.6172613834984, Tokens per sec: 83851.20587976817, Loss: 2.2590487003326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7881 , TFLOPS: 101.57211113709144, Tokens per sec: 82997.1867088682, Loss: 2.2762694358825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7882 , TFLOPS: 102.38849037555626, Tokens per sec: 83664.271200089, Loss: 2.2512621879577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7883 , TFLOPS: 102.43161866081695, Tokens per sec: 83699.51243219634, Loss: 2.287038803100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7884 , TFLOPS: 100.26445193372908, Tokens per sec: 81928.66471165825, Loss: 2.280216932296753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7885 , TFLOPS: 103.32036749452398, Tokens per sec: 84425.73198264865, Loss: 2.271040439605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7886 , TFLOPS: 103.27355038781215, Tokens per sec: 84387.47651957488, Loss: 2.2699460983276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7887 , TFLOPS: 102.75090468175736, Tokens per sec: 83960.40925906012, Loss: 2.269360303878784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7888 , TFLOPS: 101.07378581951846, Tokens per sec: 82589.99226384444, Loss: 2.261042594909668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7889 , TFLOPS: 103.34209505638303, Tokens per sec: 84443.48613276088, Loss: 2.285794734954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7890 , TFLOPS: 103.45981269947558, Tokens per sec: 84539.67624925352, Loss: 2.2721290588378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7891 , TFLOPS: 100.91358827942109, Tokens per sec: 82459.09072997932, Loss: 2.2704341411590576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7892 , TFLOPS: 102.15665814124813, Tokens per sec: 83474.83511354319, Loss: 2.265641212463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7893 , TFLOPS: 102.04045005265849, Tokens per sec: 83379.87849289428, Loss: 2.27781081199646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7894 , TFLOPS: 101.55505900008335, Tokens per sec: 82983.25296875784, Loss: 2.2667627334594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7895 , TFLOPS: 101.94541333060566, Tokens per sec: 83302.22154084205, Loss: 2.2668964862823486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7896 , TFLOPS: 103.43427613725633, Tokens per sec: 84518.8096668946, Loss: 2.244633197784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7897 , TFLOPS: 100.29028721483719, Tokens per sec: 81949.775384911, Loss: 2.259267807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7898 , TFLOPS: 102.79903009970063, Tokens per sec: 83999.73377692004, Loss: 2.259951114654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7899 , TFLOPS: 99.41052246737732, Tokens per sec: 81230.89696260219, Loss: 2.2722880840301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7900 , TFLOPS: 102.80418463216112, Tokens per sec: 84003.94567808298, Loss: 2.272345542907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7901 , TFLOPS: 100.5415269130062, Tokens per sec: 82155.06980977008, Loss: 2.284635543823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7902 , TFLOPS: 101.57376239719635, Tokens per sec: 82998.53599600714, Loss: 2.2707128524780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7903 , TFLOPS: 100.47633374024318, Tokens per sec: 82101.79879008385, Loss: 2.2878594398498535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7904 , TFLOPS: 101.64695807379181, Tokens per sec: 83058.34606757763, Loss: 2.2716753482818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7905 , TFLOPS: 101.19006566116478, Tokens per sec: 82685.00751576305, Loss: 2.270204782485962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7906 , TFLOPS: 99.72674380209403, Tokens per sec: 81489.28955546053, Loss: 2.2766530513763428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7907 , TFLOPS: 101.80254590462413, Tokens per sec: 83185.4808893378, Loss: 2.2830922603607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7908 , TFLOPS: 101.07994504181269, Tokens per sec: 82595.02512293354, Loss: 2.2539267539978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7909 , TFLOPS: 102.22961069017349, Tokens per sec: 83534.44651923577, Loss: 2.254765510559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7910 , TFLOPS: 102.16932407829437, Tokens per sec: 83485.1847767539, Loss: 2.267634153366089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7911 , TFLOPS: 102.73064831185545, Tokens per sec: 83943.85725776793, Loss: 2.2443220615386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7912 , TFLOPS: 101.44196092245127, Tokens per sec: 82890.83761812114, Loss: 2.2889344692230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7913 , TFLOPS: 102.90824248772849, Tokens per sec: 84088.97403055457, Loss: 2.2699031829833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7914 , TFLOPS: 101.5419756729219, Tokens per sec: 82972.5622453394, Loss: 2.270768404006958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7915 , TFLOPS: 101.82613979288679, Tokens per sec: 83204.76006279794, Loss: 2.2751646041870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7916 , TFLOPS: 100.09366032600533, Tokens per sec: 81789.10649242008, Loss: 2.271751880645752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7917 , TFLOPS: 101.85903542200779, Tokens per sec: 83231.63992816153, Loss: 2.271512031555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7918 , TFLOPS: 102.7763972577291, Tokens per sec: 83981.23989912392, Loss: 2.2754623889923096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7919 , TFLOPS: 102.33919726098057, Tokens per sec: 83623.99252725155, Loss: 2.301774024963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7920 , TFLOPS: 102.23529687398741, Tokens per sec: 83539.09284640553, Loss: 2.2620673179626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7921 , TFLOPS: 102.38496674825954, Tokens per sec: 83661.39195351861, Loss: 2.2702906131744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7922 , TFLOPS: 101.06916437291157, Tokens per sec: 82586.21596085533, Loss: 2.2895896434783936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7923 , TFLOPS: 103.32128911141314, Tokens per sec: 84426.48505953331, Loss: 2.2752013206481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7924 , TFLOPS: 102.74545390507419, Tokens per sec: 83955.95528911681, Loss: 2.2540998458862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7925 , TFLOPS: 102.66379633017402, Tokens per sec: 83889.23078261293, Loss: 2.2687673568725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7926 , TFLOPS: 100.7563417635858, Tokens per sec: 82330.60055400488, Loss: 2.249690055847168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7927 , TFLOPS: 102.80899462478594, Tokens per sec: 84007.87604687693, Loss: 2.2837636470794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7928 , TFLOPS: 103.2894807123629, Tokens per sec: 84400.49359784833, Loss: 2.2950057983398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7929 , TFLOPS: 101.14355828031141, Tokens per sec: 82647.00513766128, Loss: 2.2788424491882324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7930 , TFLOPS: 102.70947872952229, Tokens per sec: 83926.55904708998, Loss: 2.2792115211486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7931 , TFLOPS: 101.26774212239148, Tokens per sec: 82748.4789518014, Loss: 2.249260425567627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7932 , TFLOPS: 100.6363303931448, Tokens per sec: 82232.53617385004, Loss: 2.2759058475494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7933 , TFLOPS: 102.6855080350612, Tokens per sec: 83906.97197558524, Loss: 2.2645864486694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7934 , TFLOPS: 103.32640261621607, Tokens per sec: 84430.66343594111, Loss: 2.269432306289673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7935 , TFLOPS: 100.29853721092225, Tokens per sec: 81956.51666908589, Loss: 2.2852377891540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7936 , TFLOPS: 102.65589767244593, Tokens per sec: 83882.77658605376, Loss: 2.2651171684265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7937 , TFLOPS: 99.75343897061445, Tokens per sec: 81511.10286485335, Loss: 2.249140977859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7938 , TFLOPS: 102.85717548090383, Tokens per sec: 84047.24586469645, Loss: 2.2579684257507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7939 , TFLOPS: 101.16500651335652, Tokens per sec: 82664.53104100908, Loss: 2.2570457458496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7940 , TFLOPS: 102.15393885449166, Tokens per sec: 83472.61311433381, Loss: 2.3002307415008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7941 , TFLOPS: 100.34786004331507, Tokens per sec: 81996.81962511672, Loss: 2.256340265274048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7942 , TFLOPS: 101.64763474986636, Tokens per sec: 83058.89899701734, Loss: 2.2845242023468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7943 , TFLOPS: 102.78212271360256, Tokens per sec: 83985.91831650447, Loss: 2.309596300125122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7944 , TFLOPS: 99.48297499437375, Tokens per sec: 81290.09978750494, Loss: 2.263174057006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7945 , TFLOPS: 102.3092680085729, Tokens per sec: 83599.53656466174, Loss: 2.2831146717071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7946 , TFLOPS: 102.28139379188094, Tokens per sec: 83576.75982465656, Loss: 2.27413010597229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7947 , TFLOPS: 101.46728285913944, Tokens per sec: 82911.52882443368, Loss: 2.2505273818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7948 , TFLOPS: 102.26989264521865, Tokens per sec: 83567.36194165295, Loss: 2.245276689529419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7949 , TFLOPS: 101.97013875263775, Tokens per sec: 83322.42531967345, Loss: 2.282773494720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7950 , TFLOPS: 101.95182403239984, Tokens per sec: 83307.4598903042, Loss: 2.2695698738098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7951 , TFLOPS: 101.61930139256084, Tokens per sec: 83035.7470813975, Loss: 2.2621116638183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7952 , TFLOPS: 102.30406425717058, Tokens per sec: 83595.28444543443, Loss: 2.258944034576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7953 , TFLOPS: 101.66103446988315, Tokens per sec: 83069.84825318253, Loss: 2.285365581512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7954 , TFLOPS: 100.24539991745861, Tokens per sec: 81913.09681872113, Loss: 2.297377347946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7955 , TFLOPS: 101.37363227045344, Tokens per sec: 82835.00451763773, Loss: 2.281367063522339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7956 , TFLOPS: 102.6463050650835, Tokens per sec: 83874.93821964224, Loss: 2.275592088699341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7957 , TFLOPS: 101.66413402162705, Tokens per sec: 83072.38097670204, Loss: 2.265702486038208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7958 , TFLOPS: 100.9640786312091, Tokens per sec: 82500.3476961626, Loss: 2.2582242488861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7959 , TFLOPS: 102.93028302954244, Tokens per sec: 84106.98392464477, Loss: 2.2483468055725098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7960 , TFLOPS: 100.80338700603723, Tokens per sec: 82369.04243266422, Loss: 2.2891974449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7961 , TFLOPS: 102.73981952779461, Tokens per sec: 83951.35129439991, Loss: 2.2637856006622314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7962 , TFLOPS: 102.82397881799419, Tokens per sec: 84020.12001687485, Loss: 2.284146785736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7963 , TFLOPS: 102.6452588697417, Tokens per sec: 83874.08334649698, Loss: 2.25840163230896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7964 , TFLOPS: 101.66536031124836, Tokens per sec: 83073.38300951912, Loss: 2.2646095752716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7965 , TFLOPS: 102.71628602002694, Tokens per sec: 83932.12145939656, Loss: 2.2658159732818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7966 , TFLOPS: 103.46191901903563, Tokens per sec: 84541.39737718753, Loss: 2.254418134689331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7967 , TFLOPS: 101.0629926034237, Tokens per sec: 82581.17284913131, Loss: 2.266077756881714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7968 , TFLOPS: 102.07229796588368, Tokens per sec: 83405.9022426288, Loss: 2.283259868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7969 , TFLOPS: 100.93209914568929, Tokens per sec: 82474.21643531915, Loss: 2.2765021324157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7970 , TFLOPS: 102.0746657283626, Tokens per sec: 83407.83700230195, Loss: 2.277818202972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7971 , TFLOPS: 102.08362681816392, Tokens per sec: 83415.15933945758, Loss: 2.2483057975769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7972 , TFLOPS: 102.85496556024239, Tokens per sec: 84045.44008163556, Loss: 2.27485728263855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7973 , TFLOPS: 100.3120972775976, Tokens per sec: 81967.59694863348, Loss: 2.2842905521392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7974 , TFLOPS: 100.74630057165298, Tokens per sec: 82322.3956375933, Loss: 2.272965669631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7975 , TFLOPS: 100.05362771282839, Tokens per sec: 81756.39481366205, Loss: 2.2858381271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7976 , TFLOPS: 102.43343540129254, Tokens per sec: 83700.996937606, Loss: 2.2727818489074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7977 , TFLOPS: 101.58489240741372, Tokens per sec: 83007.6306138677, Loss: 2.2403969764709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7978 , TFLOPS: 100.92386342175816, Tokens per sec: 82467.48681328863, Loss: 2.2923920154571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7979 , TFLOPS: 101.65174472893257, Tokens per sec: 83062.25736671261, Loss: 2.2661099433898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7980 , TFLOPS: 102.28057047920356, Tokens per sec: 83576.08707467392, Loss: 2.275643825531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7981 , TFLOPS: 102.28455767744889, Tokens per sec: 83579.3451169998, Loss: 2.2641050815582275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7982 , TFLOPS: 100.54536564740273, Tokens per sec: 82158.2065384633, Loss: 2.2682201862335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7983 , TFLOPS: 102.05598524911721, Tokens per sec: 83392.57270183224, Loss: 2.2775940895080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7984 , TFLOPS: 102.3108904330637, Tokens per sec: 83600.8622894781, Loss: 2.2822368144989014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7985 , TFLOPS: 101.33007616083712, Tokens per sec: 82799.41369923623, Loss: 2.260235071182251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7986 , TFLOPS: 101.49227520605378, Tokens per sec: 82931.95071445763, Loss: 2.2672083377838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7987 , TFLOPS: 102.68504815666209, Tokens per sec: 83906.596197107, Loss: 2.2653231620788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7988 , TFLOPS: 100.64367064317997, Tokens per sec: 82238.5340811086, Loss: 2.2638180255889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7989 , TFLOPS: 103.31455686887851, Tokens per sec: 84420.98397085493, Loss: 2.258748769760132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7990 , TFLOPS: 100.70602923675297, Tokens per sec: 82289.4888931701, Loss: 2.257333755493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7991 , TFLOPS: 101.80549109564447, Tokens per sec: 83187.88748073649, Loss: 2.297285318374634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7992 , TFLOPS: 101.16636745562997, Tokens per sec: 82665.64310197449, Loss: 2.271998643875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7993 , TFLOPS: 102.02781090555452, Tokens per sec: 83369.55072043501, Loss: 2.2851476669311523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7994 , TFLOPS: 101.54796238567619, Tokens per sec: 82977.45414244264, Loss: 2.2715072631835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7995 , TFLOPS: 101.99174584616621, Tokens per sec: 83340.08103201161, Loss: 2.2704524993896484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7996 , TFLOPS: 103.33428973201669, Tokens per sec: 84437.10820129422, Loss: 2.246208667755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7997 , TFLOPS: 101.01663769716478, Tokens per sec: 82543.2950619457, Loss: 2.274331569671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7998 , TFLOPS: 100.79349924390947, Tokens per sec: 82360.96288769599, Loss: 2.2718582153320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7999 , TFLOPS: 101.99172810759461, Tokens per sec: 83340.06653736813, Loss: 2.2749063968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8000 , TFLOPS: 102.02063374262332, Tokens per sec: 83363.68607584733, Loss: 2.2717370986938477 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/optimizer.pt +[rank0]:[2024-08-28 09:32:23,420] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007722498005023226, 'preprocessing_with_comm': 0.0015250149881467223, 'state_converting': 3.3697280769993085, : 3.3806237440148834}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000, took 16.00s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006000 + eval ppl=7.535281181335449, eval loss=2.0195960998535156 +------------------------------------------------------------------ +iteration: 8001 , TFLOPS: 101.3793283628857, Tokens per sec: 82839.6589413946, Loss: 2.2793831825256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8002 , TFLOPS: 100.9456650089673, Tokens per sec: 82485.30145142, Loss: 2.258504629135132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8003 , TFLOPS: 100.4285504505774, Tokens per sec: 82062.75383404664, Loss: 2.2701520919799805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8004 , TFLOPS: 100.6604315828285, Tokens per sec: 82252.22987636039, Loss: 2.2752060890197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8005 , TFLOPS: 101.71996795714276, Tokens per sec: 83118.00432270528, Loss: 2.28837513923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8006 , TFLOPS: 102.55798758837909, Tokens per sec: 83802.77173593295, Loss: 2.2881264686584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8007 , TFLOPS: 101.0909169787191, Tokens per sec: 82603.99057502254, Loss: 2.286618709564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8008 , TFLOPS: 102.76704397564404, Tokens per sec: 83973.59709155731, Loss: 2.2877347469329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8009 , TFLOPS: 102.10954463257984, Tokens per sec: 83436.33745280073, Loss: 2.281863212585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8010 , TFLOPS: 102.74499312123568, Tokens per sec: 83955.57877078075, Loss: 2.2679407596588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8011 , TFLOPS: 101.94191461224378, Tokens per sec: 83299.36264800352, Loss: 2.283475160598755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8012 , TFLOPS: 102.00757902971851, Tokens per sec: 83353.01873387462, Loss: 2.280168056488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8013 , TFLOPS: 101.90338995499258, Tokens per sec: 83267.88315885066, Loss: 2.258589029312134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8014 , TFLOPS: 101.76624349424324, Tokens per sec: 83155.8172553084, Loss: 2.256298065185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8015 , TFLOPS: 101.98785551448333, Tokens per sec: 83336.90214184749, Loss: 2.2639026641845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8016 , TFLOPS: 101.46398042027784, Tokens per sec: 82908.83031662753, Loss: 2.2676262855529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8017 , TFLOPS: 102.67307766602065, Tokens per sec: 83896.81480105604, Loss: 2.2589211463928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8018 , TFLOPS: 100.44284839387244, Tokens per sec: 82074.43705157482, Loss: 2.2814695835113525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8019 , TFLOPS: 102.71285034549474, Tokens per sec: 83929.31408129419, Loss: 2.273855447769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8020 , TFLOPS: 102.16521630586917, Tokens per sec: 83481.8282101618, Loss: 2.277400255203247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8021 , TFLOPS: 102.7543290079203, Tokens per sec: 83963.20736411786, Loss: 2.272218704223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8022 , TFLOPS: 101.02714290773707, Tokens per sec: 82551.8791399325, Loss: 2.2768023014068604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8023 , TFLOPS: 102.14315126597424, Tokens per sec: 83463.798298059, Loss: 2.2686614990234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8024 , TFLOPS: 102.34789808763136, Tokens per sec: 83631.1021966871, Loss: 2.271087169647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8025 , TFLOPS: 101.80848737730754, Tokens per sec: 83190.33581961453, Loss: 2.2463600635528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8026 , TFLOPS: 101.61826157643429, Tokens per sec: 83034.89742087325, Loss: 2.2602336406707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8027 , TFLOPS: 100.90238107654231, Tokens per sec: 82449.933036008, Loss: 2.2661490440368652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8028 , TFLOPS: 103.45386298292804, Tokens per sec: 84534.81458270321, Loss: 2.258531332015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8029 , TFLOPS: 101.62538114154793, Tokens per sec: 83040.71500080152, Loss: 2.2541253566741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8030 , TFLOPS: 102.29293504387692, Tokens per sec: 83586.19047876075, Loss: 2.2643203735351562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8031 , TFLOPS: 102.24260342216364, Tokens per sec: 83545.0632150076, Loss: 2.267575979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8032 , TFLOPS: 102.73148020746335, Tokens per sec: 83944.53702108402, Loss: 2.258887767791748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8033 , TFLOPS: 100.67156334308221, Tokens per sec: 82261.32592422071, Loss: 2.2983057498931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8034 , TFLOPS: 102.2397998273591, Tokens per sec: 83542.77232552187, Loss: 2.282653331756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8035 , TFLOPS: 100.85920607366346, Tokens per sec: 82414.65362973225, Loss: 2.284069776535034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8036 , TFLOPS: 101.53416402195985, Tokens per sec: 82966.17914424854, Loss: 2.2778992652893066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8037 , TFLOPS: 101.16705373502447, Tokens per sec: 82666.20387853413, Loss: 2.2677700519561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8038 , TFLOPS: 101.38077786540022, Tokens per sec: 82840.84336721273, Loss: 2.2833454608917236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8039 , TFLOPS: 101.65979292668094, Tokens per sec: 83068.83375627184, Loss: 2.250028610229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8040 , TFLOPS: 101.18652346745526, Tokens per sec: 82682.11309809882, Loss: 2.2855887413024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8041 , TFLOPS: 101.57761034736298, Tokens per sec: 83001.68025514334, Loss: 2.263648271560669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8042 , TFLOPS: 99.88856231715417, Tokens per sec: 81621.51562969534, Loss: 2.2642323970794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8043 , TFLOPS: 101.72174723844063, Tokens per sec: 83119.45821925656, Loss: 2.2651288509368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8044 , TFLOPS: 103.40527371362333, Tokens per sec: 84495.11103995526, Loss: 2.2816505432128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8045 , TFLOPS: 100.50983388924583, Tokens per sec: 82129.17262420447, Loss: 2.255924940109253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8046 , TFLOPS: 102.76666307893291, Tokens per sec: 83973.28585104948, Loss: 2.2663142681121826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8047 , TFLOPS: 103.390984227572, Tokens per sec: 84483.43473305862, Loss: 2.264275550842285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8048 , TFLOPS: 102.1179853387845, Tokens per sec: 83443.23457112366, Loss: 2.2854301929473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8049 , TFLOPS: 102.73221418420987, Tokens per sec: 83945.13677237785, Loss: 2.2437050342559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8050 , TFLOPS: 101.40035083842547, Tokens per sec: 82856.8369473248, Loss: 2.278306245803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8051 , TFLOPS: 102.59602080117595, Tokens per sec: 83833.84965317127, Loss: 2.2856101989746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8052 , TFLOPS: 100.9776494097357, Tokens per sec: 82511.43672863959, Loss: 2.2789466381073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8053 , TFLOPS: 102.82053619865519, Tokens per sec: 84017.30696399204, Loss: 2.2577786445617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8054 , TFLOPS: 101.26615345005445, Tokens per sec: 82747.18080673873, Loss: 2.2708704471588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8055 , TFLOPS: 102.553806759577, Tokens per sec: 83799.35547309466, Loss: 2.2554945945739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8056 , TFLOPS: 100.28976084070243, Tokens per sec: 81949.3452710554, Loss: 2.2567262649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8057 , TFLOPS: 102.75822461431876, Tokens per sec: 83966.39056438787, Loss: 2.2782514095306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8058 , TFLOPS: 101.63299349174882, Tokens per sec: 83046.93524810999, Loss: 2.2702319622039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8059 , TFLOPS: 102.00325464230087, Tokens per sec: 83349.48516559602, Loss: 2.269080638885498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8060 , TFLOPS: 100.90050199371046, Tokens per sec: 82448.3975890542, Loss: 2.274496078491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8061 , TFLOPS: 102.08890891475924, Tokens per sec: 83419.47547655887, Loss: 2.264143705368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8062 , TFLOPS: 102.35545208823184, Tokens per sec: 83637.27476503504, Loss: 2.254688262939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8063 , TFLOPS: 101.86385891988068, Tokens per sec: 83235.58133244135, Loss: 2.258495569229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8064 , TFLOPS: 101.63731321558458, Tokens per sec: 83050.46500565577, Loss: 2.288717031478882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8065 , TFLOPS: 101.60340455165493, Tokens per sec: 83022.75736347219, Loss: 2.284511089324951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8066 , TFLOPS: 103.40328829373256, Tokens per sec: 84493.48870225325, Loss: 2.253436326980591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8067 , TFLOPS: 101.71873893197294, Tokens per sec: 83117.0000546011, Loss: 2.2727036476135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8068 , TFLOPS: 101.46447784897228, Tokens per sec: 82909.23677841871, Loss: 2.274203300476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8069 , TFLOPS: 102.12264522394645, Tokens per sec: 83447.0422832456, Loss: 2.2733755111694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8070 , TFLOPS: 102.58463068443584, Tokens per sec: 83824.54249557522, Loss: 2.2640674114227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8071 , TFLOPS: 101.36551917730259, Tokens per sec: 82828.37510037457, Loss: 2.2688660621643066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8072 , TFLOPS: 102.17800413807127, Tokens per sec: 83492.27747704223, Loss: 2.2513532638549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8073 , TFLOPS: 100.84098910775678, Tokens per sec: 82399.76807794347, Loss: 2.2541234493255615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8074 , TFLOPS: 101.50409413078395, Tokens per sec: 82941.60826210085, Loss: 2.266162395477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8075 , TFLOPS: 100.76865043238874, Tokens per sec: 82340.6582841371, Loss: 2.2481131553649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8076 , TFLOPS: 102.10428817072082, Tokens per sec: 83432.04226250235, Loss: 2.2699027061462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8077 , TFLOPS: 101.59557561056393, Tokens per sec: 83016.36013417186, Loss: 2.2625722885131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8078 , TFLOPS: 101.05486971810568, Tokens per sec: 82574.53543043628, Loss: 2.2553913593292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8079 , TFLOPS: 100.74467991713819, Tokens per sec: 82321.07135906989, Loss: 2.262843132019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8080 , TFLOPS: 100.78826993071831, Tokens per sec: 82356.689881273, Loss: 2.2663426399230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8081 , TFLOPS: 101.74138941384814, Tokens per sec: 83135.50834641662, Loss: 2.269409418106079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8082 , TFLOPS: 103.413691077953, Tokens per sec: 84501.98908503127, Loss: 2.2773189544677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8083 , TFLOPS: 100.32855982199315, Tokens per sec: 81981.04891744269, Loss: 2.272376537322998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8084 , TFLOPS: 102.84857566902727, Tokens per sec: 84040.21873703325, Loss: 2.2670867443084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8085 , TFLOPS: 103.42111621057803, Tokens per sec: 84508.05635203909, Loss: 2.260110855102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8086 , TFLOPS: 102.05102041830386, Tokens per sec: 83388.51581076853, Loss: 2.2584762573242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8087 , TFLOPS: 102.83103675382166, Tokens per sec: 84025.8872379269, Loss: 2.267397165298462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8088 , TFLOPS: 102.01007042222847, Tokens per sec: 83355.05451482863, Loss: 2.2628657817840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8089 , TFLOPS: 101.99233629521156, Tokens per sec: 83340.56350312609, Loss: 2.280775308609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8090 , TFLOPS: 102.1201924760225, Tokens per sec: 83445.0380797776, Loss: 2.273501396179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8091 , TFLOPS: 102.61571565807571, Tokens per sec: 83849.94282773478, Loss: 2.276989221572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8092 , TFLOPS: 101.28440209738136, Tokens per sec: 82762.09224623154, Loss: 2.2716495990753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8093 , TFLOPS: 102.54109903908727, Tokens per sec: 83788.97166755667, Loss: 2.279069423675537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8094 , TFLOPS: 100.31872995582226, Tokens per sec: 81973.01668075119, Loss: 2.2676339149475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8095 , TFLOPS: 102.76206882851106, Tokens per sec: 83969.53177076328, Loss: 2.2739267349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8096 , TFLOPS: 101.34916931143854, Tokens per sec: 82815.01520409407, Loss: 2.2583398818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8097 , TFLOPS: 102.40002046048369, Tokens per sec: 83673.692729294, Loss: 2.287824869155884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8098 , TFLOPS: 100.33170930255902, Tokens per sec: 81983.6224390879, Loss: 2.2541961669921875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8099 , TFLOPS: 101.6122522317112, Tokens per sec: 83029.98703060571, Loss: 2.264350175857544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8100 , TFLOPS: 102.17123207128304, Tokens per sec: 83486.74384694106, Loss: 2.2843050956726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8101 , TFLOPS: 100.68220902920217, Tokens per sec: 82270.02478839365, Loss: 2.2564921379089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8102 , TFLOPS: 101.62193830829106, Tokens per sec: 83037.90177312092, Loss: 2.2644667625427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8103 , TFLOPS: 101.57531703384677, Tokens per sec: 82999.80632963411, Loss: 2.2665393352508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8104 , TFLOPS: 103.35440306972806, Tokens per sec: 84453.54332730157, Loss: 2.242953300476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8105 , TFLOPS: 101.63840656530063, Tokens per sec: 83051.35840985407, Loss: 2.2787163257598877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8106 , TFLOPS: 102.23373478674577, Tokens per sec: 83537.81642470868, Loss: 2.2999813556671143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8107 , TFLOPS: 101.9510454212407, Tokens per sec: 83306.82366708285, Loss: 2.277348756790161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8108 , TFLOPS: 102.54092349014167, Tokens per sec: 83788.82822199419, Loss: 2.2718796730041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8109 , TFLOPS: 101.51152090590635, Tokens per sec: 82947.67687123557, Loss: 2.269832134246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8110 , TFLOPS: 101.54429079279714, Tokens per sec: 82974.45398938595, Loss: 2.280764102935791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8111 , TFLOPS: 100.90659593276835, Tokens per sec: 82453.37710353035, Loss: 2.274317502975464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8112 , TFLOPS: 101.78909254371415, Tokens per sec: 83174.48780181793, Loss: 2.260006904602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8113 , TFLOPS: 101.56770218003254, Tokens per sec: 82993.58403655872, Loss: 2.241853713989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8114 , TFLOPS: 102.1871395048698, Tokens per sec: 83499.74222041857, Loss: 2.2588818073272705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8115 , TFLOPS: 101.64098780033886, Tokens per sec: 83053.46760345071, Loss: 2.2586467266082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8116 , TFLOPS: 100.45669222256659, Tokens per sec: 82085.74920037267, Loss: 2.2735188007354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8117 , TFLOPS: 101.39507940298624, Tokens per sec: 82852.52952172859, Loss: 2.2526943683624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8118 , TFLOPS: 100.63497879202355, Tokens per sec: 82231.43174578054, Loss: 2.2644810676574707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8119 , TFLOPS: 101.74259769826088, Tokens per sec: 83136.49566671427, Loss: 2.262120485305786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8120 , TFLOPS: 103.30343935013752, Tokens per sec: 84411.89956010127, Loss: 2.3017702102661133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8121 , TFLOPS: 101.01231199771117, Tokens per sec: 82539.76042156873, Loss: 2.274812936782837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8122 , TFLOPS: 102.76675570293636, Tokens per sec: 83973.36153650707, Loss: 2.2562193870544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8123 , TFLOPS: 103.39214588581717, Tokens per sec: 84484.3839539145, Loss: 2.2754926681518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8124 , TFLOPS: 102.77315288995945, Tokens per sec: 83978.58884270188, Loss: 2.2485404014587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8125 , TFLOPS: 102.14135252079043, Tokens per sec: 83462.3284970655, Loss: 2.2685296535491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8126 , TFLOPS: 102.07116722332101, Tokens per sec: 83404.97828377294, Loss: 2.2686972618103027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8127 , TFLOPS: 102.84397239906949, Tokens per sec: 84036.45728665203, Loss: 2.244036912918091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8128 , TFLOPS: 101.31171707576294, Tokens per sec: 82784.41201821723, Loss: 2.2637858390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8129 , TFLOPS: 102.65661945773692, Tokens per sec: 83883.36637539543, Loss: 2.2788515090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8130 , TFLOPS: 101.38894807066292, Tokens per sec: 82847.51945225267, Loss: 2.2575111389160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8131 , TFLOPS: 101.68875553657023, Tokens per sec: 83092.49985037629, Loss: 2.2710235118865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8132 , TFLOPS: 101.61358029000071, Tokens per sec: 83031.07222122134, Loss: 2.2636337280273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8133 , TFLOPS: 102.00535514526443, Tokens per sec: 83351.2015406393, Loss: 2.2905590534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8134 , TFLOPS: 101.62543815663886, Tokens per sec: 83040.76158930002, Loss: 2.2593510150909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8135 , TFLOPS: 101.99608516422514, Tokens per sec: 83343.62680050154, Loss: 2.262809991836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8136 , TFLOPS: 101.40928312564169, Tokens per sec: 82864.13574914672, Loss: 2.2683279514312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8137 , TFLOPS: 100.84684692894821, Tokens per sec: 82404.55465443274, Loss: 2.2590084075927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8138 , TFLOPS: 102.22876149242275, Tokens per sec: 83533.75261789335, Loss: 2.249098539352417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8139 , TFLOPS: 100.5736187640982, Tokens per sec: 82181.29288741511, Loss: 2.264580488204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8140 , TFLOPS: 102.3065468167325, Tokens per sec: 83597.31300875924, Loss: 2.2828922271728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8141 , TFLOPS: 100.52401489374734, Tokens per sec: 82140.76028803419, Loss: 2.2735626697540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8142 , TFLOPS: 102.66284442750893, Tokens per sec: 83888.4529584411, Loss: 2.2858645915985107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8143 , TFLOPS: 101.52128760904203, Tokens per sec: 82955.65749578502, Loss: 2.285691976547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8144 , TFLOPS: 102.26698442249959, Tokens per sec: 83564.9855580049, Loss: 2.2500298023223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8145 , TFLOPS: 101.98831435867272, Tokens per sec: 83337.27707524635, Loss: 2.2682976722717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8146 , TFLOPS: 102.71758091117589, Tokens per sec: 83933.17954828784, Loss: 2.254654884338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8147 , TFLOPS: 100.97207122124777, Tokens per sec: 82506.87864723075, Loss: 2.264735698699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8148 , TFLOPS: 102.22489260335385, Tokens per sec: 83530.59124904108, Loss: 2.253523349761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8149 , TFLOPS: 100.6110960687591, Tokens per sec: 82211.91655780516, Loss: 2.2584445476531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8150 , TFLOPS: 101.71600992341753, Tokens per sec: 83114.77011145957, Loss: 2.2641847133636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8151 , TFLOPS: 101.27021366012065, Tokens per sec: 82750.49850890294, Loss: 2.2905545234680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8152 , TFLOPS: 102.21079598104524, Tokens per sec: 83519.07253607339, Loss: 2.2653284072875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8153 , TFLOPS: 102.26687910939955, Tokens per sec: 83564.89950395985, Loss: 2.2651753425598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8154 , TFLOPS: 101.12467803280622, Tokens per sec: 82631.57760140399, Loss: 2.268031120300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8155 , TFLOPS: 102.00979629021609, Tokens per sec: 83354.83051450455, Loss: 2.2681875228881836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8156 , TFLOPS: 100.25961036422648, Tokens per sec: 81924.7085405846, Loss: 2.2601478099823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8157 , TFLOPS: 101.6467240102084, Tokens per sec: 83058.15480819829, Loss: 2.262259006500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8158 , TFLOPS: 102.64791791920646, Tokens per sec: 83876.25612426466, Loss: 2.2769041061401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8159 , TFLOPS: 101.15697978869368, Tokens per sec: 82657.97219766078, Loss: 2.2618420124053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8160 , TFLOPS: 102.13693136634653, Tokens per sec: 83458.71585795874, Loss: 2.262756109237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8161 , TFLOPS: 103.35500111176813, Tokens per sec: 84454.03200284747, Loss: 2.284485340118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8162 , TFLOPS: 102.62612539883847, Tokens per sec: 83858.44889488259, Loss: 2.2793664932250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8163 , TFLOPS: 102.13838431769535, Tokens per sec: 83459.90310190816, Loss: 2.2636361122131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8164 , TFLOPS: 101.93984117884621, Tokens per sec: 83297.66839219944, Loss: 2.277820110321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8165 , TFLOPS: 102.68263386833057, Tokens per sec: 83904.62341996201, Loss: 2.2664756774902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8166 , TFLOPS: 101.36816899153574, Tokens per sec: 82830.54033179677, Loss: 2.25947904586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8167 , TFLOPS: 102.69687003988507, Tokens per sec: 83916.25616221083, Loss: 2.257723808288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8168 , TFLOPS: 100.6396863665491, Tokens per sec: 82235.27842610911, Loss: 2.264647960662842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8169 , TFLOPS: 102.7410327888271, Tokens per sec: 83952.34268122168, Loss: 2.2678704261779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8170 , TFLOPS: 100.93309430858808, Tokens per sec: 82475.02960854153, Loss: 2.2554237842559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8171 , TFLOPS: 102.62394436449851, Tokens per sec: 83856.6667155786, Loss: 2.2878832817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8172 , TFLOPS: 100.70577201654567, Tokens per sec: 82289.27871191729, Loss: 2.261171579360962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8173 , TFLOPS: 102.7846606250838, Tokens per sec: 83987.99210930744, Loss: 2.241870164871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8174 , TFLOPS: 100.76924394992935, Tokens per sec: 82341.14326259801, Loss: 2.268554449081421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8175 , TFLOPS: 101.52223857789616, Tokens per sec: 82956.43455691585, Loss: 2.2605886459350586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8176 , TFLOPS: 101.53447779399353, Tokens per sec: 82966.43553545447, Loss: 2.275111675262451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8177 , TFLOPS: 101.06134416230933, Tokens per sec: 82579.82586546209, Loss: 2.261906147003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8178 , TFLOPS: 101.68904770478555, Tokens per sec: 83092.73858854611, Loss: 2.2729177474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8179 , TFLOPS: 101.14736967666484, Tokens per sec: 82650.11952773573, Loss: 2.2873334884643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8180 , TFLOPS: 103.38886999029161, Tokens per sec: 84481.70713535149, Loss: 2.2821640968322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8181 , TFLOPS: 101.57855343262784, Tokens per sec: 83002.45087439043, Loss: 2.285336971282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8182 , TFLOPS: 102.14488239639955, Tokens per sec: 83465.21284929277, Loss: 2.2798330783843994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8183 , TFLOPS: 102.0000451196326, Tokens per sec: 83346.8625820033, Loss: 2.2657551765441895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8184 , TFLOPS: 102.71632360659957, Tokens per sec: 83932.15217235262, Loss: 2.264711856842041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8185 , TFLOPS: 100.92929571229881, Tokens per sec: 82471.925677729, Loss: 2.262802839279175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8186 , TFLOPS: 102.30991570087598, Tokens per sec: 83600.06581071566, Loss: 2.274740219116211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8187 , TFLOPS: 100.01088974569267, Tokens per sec: 81721.4725205423, Loss: 2.2611470222473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8188 , TFLOPS: 101.60016127807982, Tokens per sec: 83020.10720114468, Loss: 2.28904390335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8189 , TFLOPS: 102.04313510326632, Tokens per sec: 83382.07251686529, Loss: 2.275942802429199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8190 , TFLOPS: 102.1068866304369, Tokens per sec: 83434.16553082666, Loss: 2.272029161453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8191 , TFLOPS: 102.11324655960307, Tokens per sec: 83439.36239266748, Loss: 2.259021520614624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8192 , TFLOPS: 101.10062535715527, Tokens per sec: 82611.92354095863, Loss: 2.277975559234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8193 , TFLOPS: 101.3297634398014, Tokens per sec: 82799.15816682778, Loss: 2.2613072395324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8194 , TFLOPS: 101.09505832835687, Tokens per sec: 82607.37457841939, Loss: 2.263542652130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8195 , TFLOPS: 101.84215118572817, Tokens per sec: 83217.84338405855, Loss: 2.2706854343414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8196 , TFLOPS: 102.80459285403933, Tokens per sec: 84004.27924668822, Loss: 2.2585973739624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8197 , TFLOPS: 101.11877712311225, Tokens per sec: 82626.75581619011, Loss: 2.274960994720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8198 , TFLOPS: 101.7382035498446, Tokens per sec: 83132.90509492789, Loss: 2.2785370349884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8199 , TFLOPS: 103.43133540276325, Tokens per sec: 84516.40671703905, Loss: 2.2738771438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8200 , TFLOPS: 102.12301763998254, Tokens per sec: 83447.34659397553, Loss: 2.275789737701416 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/optimizer.pt +[rank0]:[2024-08-28 10:25:18,294] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0077176720078568906, 'preprocessing_with_comm': 0.0016554670000914484, 'state_converting': 3.379955387994414, : 3.3910073899896815}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200, took 15.87s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006200 + eval ppl=7.781376838684082, eval loss=2.0517332553863525 +------------------------------------------------------------------ +iteration: 8201 , TFLOPS: 100.91671999892105, Tokens per sec: 82461.64973860038, Loss: 2.2582082748413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8202 , TFLOPS: 100.42866508343165, Tokens per sec: 82062.84750350277, Loss: 2.280533790588379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8203 , TFLOPS: 100.28044320757554, Tokens per sec: 81941.73159317022, Loss: 2.2764036655426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8204 , TFLOPS: 101.61757671848876, Tokens per sec: 83034.33780581615, Loss: 2.272409677505493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8205 , TFLOPS: 101.82352576332627, Tokens per sec: 83202.62407195295, Loss: 2.2878963947296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8206 , TFLOPS: 101.60063928975609, Tokens per sec: 83020.49779679047, Loss: 2.255086898803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8207 , TFLOPS: 102.21794597272594, Tokens per sec: 83524.91497833285, Loss: 2.2528789043426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8208 , TFLOPS: 101.58818870351742, Tokens per sec: 83010.32410226825, Loss: 2.2715044021606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8209 , TFLOPS: 103.29033272000659, Tokens per sec: 84401.18979522662, Loss: 2.2686052322387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8210 , TFLOPS: 102.79348745981062, Tokens per sec: 83995.20474318566, Loss: 2.2756388187408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8211 , TFLOPS: 102.84232766857751, Tokens per sec: 84035.11333502791, Loss: 2.2444112300872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8212 , TFLOPS: 102.70868576951082, Tokens per sec: 83925.911099052, Loss: 2.26650333404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8213 , TFLOPS: 103.41368379799249, Tokens per sec: 84501.98313638812, Loss: 2.258866786956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8214 , TFLOPS: 102.63734679298598, Tokens per sec: 83867.6181849049, Loss: 2.265753984451294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8215 , TFLOPS: 102.49310341583562, Tokens per sec: 83749.7532082805, Loss: 2.265695095062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8216 , TFLOPS: 100.88299574862828, Tokens per sec: 82434.09278554672, Loss: 2.2799017429351807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8217 , TFLOPS: 102.73696622853376, Tokens per sec: 83949.0197901234, Loss: 2.258439302444458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8218 , TFLOPS: 102.59028917150475, Tokens per sec: 83829.16619102166, Loss: 2.2605385780334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8219 , TFLOPS: 103.2789919661852, Tokens per sec: 84391.92297334218, Loss: 2.278964042663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8220 , TFLOPS: 102.05905853216638, Tokens per sec: 83395.08396052504, Loss: 2.2711074352264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8221 , TFLOPS: 99.34254598242644, Tokens per sec: 81175.35162687843, Loss: 2.2694528102874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8222 , TFLOPS: 101.36465021127582, Tokens per sec: 82827.66504586491, Loss: 2.2852554321289062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8223 , TFLOPS: 102.70549466534673, Tokens per sec: 83923.30356569318, Loss: 2.2511634826660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8224 , TFLOPS: 101.64070299257142, Tokens per sec: 83053.23487969233, Loss: 2.2582952976226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8225 , TFLOPS: 102.79562114193632, Tokens per sec: 83996.94822977643, Loss: 2.275787353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8226 , TFLOPS: 101.41398468642399, Tokens per sec: 82867.97751549086, Loss: 2.265155792236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8227 , TFLOPS: 100.77332026475978, Tokens per sec: 82344.47412437963, Loss: 2.2615139484405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8228 , TFLOPS: 102.75410744628643, Tokens per sec: 83963.02632040333, Loss: 2.2509396076202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8229 , TFLOPS: 102.0351020778269, Tokens per sec: 83375.50852499026, Loss: 2.2620058059692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8230 , TFLOPS: 100.30562315317133, Tokens per sec: 81962.30677490598, Loss: 2.2669107913970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8231 , TFLOPS: 102.1669705865972, Tokens per sec: 83483.26167810388, Loss: 2.263302803039551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8232 , TFLOPS: 102.72931847944221, Tokens per sec: 83942.77061747013, Loss: 2.277778148651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8233 , TFLOPS: 98.88901845195355, Tokens per sec: 80804.76260689176, Loss: 2.2695884704589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8234 , TFLOPS: 102.82556404551865, Tokens per sec: 84021.41534709248, Loss: 2.251026153564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8235 , TFLOPS: 99.74293606605895, Tokens per sec: 81502.52067117224, Loss: 2.266343593597412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8236 , TFLOPS: 102.49679938645168, Tokens per sec: 83752.77328101364, Loss: 2.2934000492095947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8237 , TFLOPS: 100.42562042664278, Tokens per sec: 82060.35963606414, Loss: 2.2774667739868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8238 , TFLOPS: 100.56291626511397, Tokens per sec: 82172.54759998825, Loss: 2.2543957233428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8239 , TFLOPS: 101.59786859829457, Tokens per sec: 83018.23379347331, Loss: 2.2718191146850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8240 , TFLOPS: 99.93199887325858, Tokens per sec: 81657.00875784468, Loss: 2.293412208557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8241 , TFLOPS: 100.93105696977547, Tokens per sec: 82473.36484655198, Loss: 2.26165771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8242 , TFLOPS: 101.61980055742761, Tokens per sec: 83036.15496185978, Loss: 2.2621169090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8243 , TFLOPS: 101.7780304147699, Tokens per sec: 83165.44865149306, Loss: 2.2776026725769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8244 , TFLOPS: 101.58852801816192, Tokens per sec: 83010.601364999, Loss: 2.261176586151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8245 , TFLOPS: 102.4218082566581, Tokens per sec: 83691.49609841585, Loss: 2.258256435394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8246 , TFLOPS: 101.09860249176977, Tokens per sec: 82610.27060557895, Loss: 2.2776010036468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8247 , TFLOPS: 102.80463846946371, Tokens per sec: 84004.31652022572, Loss: 2.2842609882354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8248 , TFLOPS: 102.65150768230767, Tokens per sec: 83879.18941210338, Loss: 2.274019718170166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8249 , TFLOPS: 102.63021083966547, Tokens per sec: 83861.78721375094, Loss: 2.2756104469299316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8250 , TFLOPS: 102.73095992607779, Tokens per sec: 83944.11188577065, Loss: 2.2584519386291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8251 , TFLOPS: 103.30517192872935, Tokens per sec: 84413.3152946692, Loss: 2.250551223754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8252 , TFLOPS: 102.6142330670339, Tokens per sec: 83848.73136442922, Loss: 2.294525146484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8253 , TFLOPS: 103.39412856068454, Tokens per sec: 84486.00404858722, Loss: 2.26517391204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8254 , TFLOPS: 100.76047440462622, Tokens per sec: 82333.97744138265, Loss: 2.262265205383301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8255 , TFLOPS: 102.4957494972469, Tokens per sec: 83751.91538951785, Loss: 2.2860426902770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8256 , TFLOPS: 102.61626310438784, Tokens per sec: 83850.39016020895, Loss: 2.2738003730773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8257 , TFLOPS: 102.63605480324415, Tokens per sec: 83866.5624668281, Loss: 2.2460503578186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8258 , TFLOPS: 101.52998844255976, Tokens per sec: 82962.76717083175, Loss: 2.2549033164978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8259 , TFLOPS: 102.81378565233614, Tokens per sec: 84011.79091882022, Loss: 2.2790822982788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8260 , TFLOPS: 102.09385492003781, Tokens per sec: 83423.5169847935, Loss: 2.2661538124084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8261 , TFLOPS: 102.10158303985878, Tokens per sec: 83429.83183043872, Loss: 2.253361225128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8262 , TFLOPS: 101.60852515874775, Tokens per sec: 83026.94154334388, Loss: 2.276200294494629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8263 , TFLOPS: 102.09698529008818, Tokens per sec: 83426.07489074455, Loss: 2.2616193294525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8264 , TFLOPS: 102.14898795544067, Tokens per sec: 83468.56760727178, Loss: 2.255826711654663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8265 , TFLOPS: 100.7801130675586, Tokens per sec: 82350.02469840886, Loss: 2.271583080291748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8266 , TFLOPS: 102.67749265987982, Tokens per sec: 83900.42240619042, Loss: 2.2544398307800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8267 , TFLOPS: 100.84236383006869, Tokens per sec: 82400.89139893305, Loss: 2.2685067653656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8268 , TFLOPS: 101.63550230040872, Tokens per sec: 83048.98526024848, Loss: 2.2697720527648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8269 , TFLOPS: 101.56920126613265, Tokens per sec: 82994.80897840104, Loss: 2.270780086517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8270 , TFLOPS: 102.8167125422116, Tokens per sec: 84014.18255588232, Loss: 2.2777462005615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8271 , TFLOPS: 99.04297398764817, Tokens per sec: 80930.56363827591, Loss: 2.2657957077026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8272 , TFLOPS: 102.92790785631493, Tokens per sec: 84105.04310945817, Loss: 2.254335641860962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8273 , TFLOPS: 99.76965860254985, Tokens per sec: 81524.35634363824, Loss: 2.2595839500427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8274 , TFLOPS: 101.39490221072948, Tokens per sec: 82852.38473337426, Loss: 2.2690846920013428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8275 , TFLOPS: 100.60347711313716, Tokens per sec: 82205.69091303712, Loss: 2.2659103870391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8276 , TFLOPS: 100.6630908005684, Tokens per sec: 82254.40279163001, Loss: 2.2819664478302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8277 , TFLOPS: 101.03486378356396, Tokens per sec: 82558.18806632368, Loss: 2.2500579357147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8278 , TFLOPS: 100.47734433204586, Tokens per sec: 82102.62457066079, Loss: 2.253525495529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8279 , TFLOPS: 101.40318087930181, Tokens per sec: 82859.14944657659, Loss: 2.2722713947296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8280 , TFLOPS: 101.60673847450282, Tokens per sec: 83025.48159765401, Loss: 2.2818751335144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8281 , TFLOPS: 102.237526089222, Tokens per sec: 83540.91439555881, Loss: 2.2672958374023438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8282 , TFLOPS: 102.14593958786821, Tokens per sec: 83466.07670765635, Loss: 2.2733547687530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8283 , TFLOPS: 101.76043095155194, Tokens per sec: 83151.06767704716, Loss: 2.2717976570129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8284 , TFLOPS: 101.50777800071079, Tokens per sec: 82944.6184470493, Loss: 2.2780630588531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8285 , TFLOPS: 102.79442348511951, Tokens per sec: 83995.96959356099, Loss: 2.293886661529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8286 , TFLOPS: 102.14993963331759, Tokens per sec: 83469.34524776335, Loss: 2.272352695465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8287 , TFLOPS: 102.12124658449098, Tokens per sec: 83445.89941894235, Loss: 2.2676119804382324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8288 , TFLOPS: 102.80023008858288, Tokens per sec: 84000.71431872618, Loss: 2.2609167098999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8289 , TFLOPS: 102.79451690289608, Tokens per sec: 83996.04592763101, Loss: 2.272810459136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8290 , TFLOPS: 102.15478772607928, Tokens per sec: 83473.30674915995, Loss: 2.275106906890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8291 , TFLOPS: 103.43403858723258, Tokens per sec: 84518.6155586551, Loss: 2.2518956661224365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8292 , TFLOPS: 101.47354874238454, Tokens per sec: 82916.64883893145, Loss: 2.265906810760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8293 , TFLOPS: 102.10262807674717, Tokens per sec: 83430.68575698185, Loss: 2.2786221504211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8294 , TFLOPS: 102.6466605541911, Tokens per sec: 83875.22869894316, Loss: 2.250518560409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8295 , TFLOPS: 103.38945639469047, Tokens per sec: 84482.1863014812, Loss: 2.271143674850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8296 , TFLOPS: 101.49169046208044, Tokens per sec: 82931.47290510437, Loss: 2.275419235229492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8297 , TFLOPS: 102.7455063914963, Tokens per sec: 83955.99817712347, Loss: 2.2949745655059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8298 , TFLOPS: 101.3335473774462, Tokens per sec: 82802.25011968456, Loss: 2.2652554512023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8299 , TFLOPS: 102.14646194685903, Tokens per sec: 83466.50354063447, Loss: 2.2797513008117676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8300 , TFLOPS: 101.69691429264023, Tokens per sec: 83099.16657998615, Loss: 2.265739917755127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8301 , TFLOPS: 102.13715692062055, Tokens per sec: 83458.90016416232, Loss: 2.29506254196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8302 , TFLOPS: 102.62976472690971, Tokens per sec: 83861.42268353426, Loss: 2.2758803367614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8303 , TFLOPS: 100.3035044572858, Tokens per sec: 81960.57553395785, Loss: 2.2872507572174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8304 , TFLOPS: 102.5857393261478, Tokens per sec: 83825.44839525704, Loss: 2.277738332748413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8305 , TFLOPS: 101.6986815786368, Tokens per sec: 83100.61067486794, Loss: 2.245363712310791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8306 , TFLOPS: 100.44108804890847, Tokens per sec: 82072.99862838947, Loss: 2.263382911682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8307 , TFLOPS: 102.15514758549983, Tokens per sec: 83473.6007995561, Loss: 2.2695562839508057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8308 , TFLOPS: 102.33101290083131, Tokens per sec: 83617.3048758894, Loss: 2.272118091583252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8309 , TFLOPS: 98.92173787381886, Tokens per sec: 80831.49848877077, Loss: 2.2954583168029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8310 , TFLOPS: 102.79233210111678, Tokens per sec: 83994.26066985534, Loss: 2.265012264251709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8311 , TFLOPS: 100.49481421656077, Tokens per sec: 82116.89966301274, Loss: 2.264314651489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8312 , TFLOPS: 101.90404721901147, Tokens per sec: 83268.42022619997, Loss: 2.263209342956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8313 , TFLOPS: 100.79573792616213, Tokens per sec: 82362.79217259344, Loss: 2.269418239593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8314 , TFLOPS: 101.06197682464058, Tokens per sec: 82580.34283013915, Loss: 2.270008087158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8315 , TFLOPS: 101.58105146317885, Tokens per sec: 83004.4920794587, Loss: 2.2866220474243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8316 , TFLOPS: 100.52319531207444, Tokens per sec: 82140.09058675135, Loss: 2.2504618167877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8317 , TFLOPS: 100.99292045447835, Tokens per sec: 82523.91509241048, Loss: 2.2536325454711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8318 , TFLOPS: 101.70985641121749, Tokens per sec: 83109.7419182354, Loss: 2.2545459270477295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8319 , TFLOPS: 101.57582437106747, Tokens per sec: 83000.22088793716, Loss: 2.268131732940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8320 , TFLOPS: 102.84151499591921, Tokens per sec: 84034.44927927876, Loss: 2.250844955444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8321 , TFLOPS: 101.6275407787459, Tokens per sec: 83042.47969594998, Loss: 2.249976634979248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8322 , TFLOPS: 101.76251484875236, Tokens per sec: 83152.77048309393, Loss: 2.274486780166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8323 , TFLOPS: 102.7244169243639, Tokens per sec: 83938.76543063838, Loss: 2.2718076705932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8324 , TFLOPS: 101.49958649722717, Tokens per sec: 82937.92495868479, Loss: 2.259591579437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8325 , TFLOPS: 103.42491113943788, Tokens per sec: 84511.15728610064, Loss: 2.273296356201172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8326 , TFLOPS: 102.74265117847499, Tokens per sec: 83953.66510906402, Loss: 2.280170202255249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8327 , TFLOPS: 103.34605224687242, Tokens per sec: 84446.7196549769, Loss: 2.2665958404541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8328 , TFLOPS: 101.52315659495366, Tokens per sec: 82957.1846922854, Loss: 2.2848284244537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8329 , TFLOPS: 103.42295038011041, Tokens per sec: 84509.55509917972, Loss: 2.28171968460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8330 , TFLOPS: 102.00056693365842, Tokens per sec: 83347.28896967649, Loss: 2.267944574356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8331 , TFLOPS: 102.11069878484324, Tokens per sec: 83437.28054032566, Loss: 2.264510154724121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8332 , TFLOPS: 102.18745274079679, Tokens per sec: 83499.99817355792, Loss: 2.251838207244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8333 , TFLOPS: 102.62933805526141, Tokens per sec: 83861.0740391469, Loss: 2.2533462047576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8334 , TFLOPS: 101.465864372139, Tokens per sec: 82910.36974219051, Loss: 2.2580206394195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8335 , TFLOPS: 102.71544659718091, Tokens per sec: 83931.43554538042, Loss: 2.265233039855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8336 , TFLOPS: 102.0816947919624, Tokens per sec: 83413.58063111355, Loss: 2.2808845043182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8337 , TFLOPS: 101.60388281259597, Tokens per sec: 83023.14816279864, Loss: 2.264126777648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8338 , TFLOPS: 102.31901774298964, Tokens per sec: 83607.50332363416, Loss: 2.268160820007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8339 , TFLOPS: 101.0442431940181, Tokens per sec: 82565.85222405325, Loss: 2.2396080493927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8340 , TFLOPS: 102.70809285139013, Tokens per sec: 83925.4266103925, Loss: 2.2576777935028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8341 , TFLOPS: 100.41523445855204, Tokens per sec: 82051.87299417859, Loss: 2.2667126655578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8342 , TFLOPS: 102.73445513781697, Tokens per sec: 83946.9679132575, Loss: 2.278421401977539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8343 , TFLOPS: 101.58930690653075, Tokens per sec: 83011.23781473555, Loss: 2.2621548175811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8344 , TFLOPS: 100.27129749213343, Tokens per sec: 81934.2583936508, Loss: 2.281035900115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8345 , TFLOPS: 101.57333553554196, Tokens per sec: 82998.18719636094, Loss: 2.2878756523132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8346 , TFLOPS: 102.46667084257744, Tokens per sec: 83728.15447223626, Loss: 2.2633919715881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8347 , TFLOPS: 99.67548678249815, Tokens per sec: 81447.4061252757, Loss: 2.2708041667938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8348 , TFLOPS: 102.12713206848755, Tokens per sec: 83450.70859942179, Loss: 2.270355701446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8349 , TFLOPS: 101.0058188059623, Tokens per sec: 82534.45466744112, Loss: 2.2529709339141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8350 , TFLOPS: 102.71202331137141, Tokens per sec: 83928.63829042224, Loss: 2.2640392780303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8351 , TFLOPS: 100.94019766015974, Tokens per sec: 82480.83393996727, Loss: 2.250230312347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8352 , TFLOPS: 100.8603303026306, Tokens per sec: 82415.57226616151, Loss: 2.28102707862854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8353 , TFLOPS: 102.64195956226142, Tokens per sec: 83871.38739742311, Loss: 2.2823920249938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8354 , TFLOPS: 100.45260332859264, Tokens per sec: 82082.40805984907, Loss: 2.262340784072876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8355 , TFLOPS: 100.97164369976213, Tokens per sec: 82506.52930841946, Loss: 2.285836935043335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8356 , TFLOPS: 100.95864152322292, Tokens per sec: 82495.90489526348, Loss: 2.2730352878570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8357 , TFLOPS: 102.25781323763398, Tokens per sec: 83557.49154676391, Loss: 2.283959150314331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8358 , TFLOPS: 102.04345496015638, Tokens per sec: 83382.333880164, Loss: 2.2681493759155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8359 , TFLOPS: 102.28951382911114, Tokens per sec: 83583.39491610572, Loss: 2.2611639499664307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8360 , TFLOPS: 100.99765545331762, Tokens per sec: 82527.78418185175, Loss: 2.2904720306396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8361 , TFLOPS: 103.34767748055627, Tokens per sec: 84448.04767525723, Loss: 2.2663486003875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8362 , TFLOPS: 101.3937985984428, Tokens per sec: 82851.4829433653, Loss: 2.262589931488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8363 , TFLOPS: 102.83953627975261, Tokens per sec: 84032.83241936234, Loss: 2.255389928817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8364 , TFLOPS: 102.17744545312223, Tokens per sec: 83491.82096118854, Loss: 2.2682580947875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8365 , TFLOPS: 102.82230215261609, Tokens per sec: 84018.7499704332, Loss: 2.285466194152832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8366 , TFLOPS: 100.54805228644668, Tokens per sec: 82160.40186038635, Loss: 2.277585029602051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8367 , TFLOPS: 103.34536060479174, Tokens per sec: 84446.15449642834, Loss: 2.2689852714538574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8368 , TFLOPS: 102.60333068073402, Tokens per sec: 83839.8227439312, Loss: 2.262784242630005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8369 , TFLOPS: 102.10563080360998, Tokens per sec: 83433.13936239854, Loss: 2.256438970565796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8370 , TFLOPS: 101.39417329219145, Tokens per sec: 82851.78911527281, Loss: 2.264146327972412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8371 , TFLOPS: 102.67080271584486, Tokens per sec: 83894.95588070496, Loss: 2.2634968757629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8372 , TFLOPS: 102.12831064096716, Tokens per sec: 83451.67164133403, Loss: 2.2680208683013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8373 , TFLOPS: 101.2268707761472, Tokens per sec: 82715.08192266313, Loss: 2.2627830505371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8374 , TFLOPS: 102.69866195850756, Tokens per sec: 83917.72038504537, Loss: 2.2709364891052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8375 , TFLOPS: 100.28860020359991, Tokens per sec: 81948.39688460148, Loss: 2.2672765254974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8376 , TFLOPS: 101.77158162920196, Tokens per sec: 83160.179182799, Loss: 2.2717254161834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8377 , TFLOPS: 101.77702028828122, Tokens per sec: 83164.62325113619, Loss: 2.2677745819091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8378 , TFLOPS: 102.92486150722274, Tokens per sec: 84102.55385919496, Loss: 2.2669742107391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8379 , TFLOPS: 99.8168050678934, Tokens per sec: 81562.88093412844, Loss: 2.2630977630615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8380 , TFLOPS: 102.66949686526732, Tokens per sec: 83893.88883658248, Loss: 2.2573306560516357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8381 , TFLOPS: 100.83711462075394, Tokens per sec: 82396.60213487528, Loss: 2.285914421081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8382 , TFLOPS: 102.0765973459077, Tokens per sec: 83409.4153767223, Loss: 2.288336753845215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8383 , TFLOPS: 101.68432318181878, Tokens per sec: 83088.87805920992, Loss: 2.2843399047851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8384 , TFLOPS: 101.46510391188531, Tokens per sec: 82909.74835054124, Loss: 2.268394708633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8385 , TFLOPS: 100.83879987055472, Tokens per sec: 82397.9791958697, Loss: 2.259237289428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8386 , TFLOPS: 101.49513547852281, Tokens per sec: 82934.28791672205, Loss: 2.270603895187378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8387 , TFLOPS: 100.91323285193353, Tokens per sec: 82458.80030102973, Loss: 2.2590880393981934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8388 , TFLOPS: 102.78979129694524, Tokens per sec: 83992.18451336067, Loss: 2.274460792541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8389 , TFLOPS: 100.9822692228397, Tokens per sec: 82515.21169685145, Loss: 2.2493889331817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8390 , TFLOPS: 102.14761514693124, Tokens per sec: 83467.44585009948, Loss: 2.277534246444702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8391 , TFLOPS: 102.20471928111526, Tokens per sec: 83514.10710813242, Loss: 2.2597286701202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8392 , TFLOPS: 100.30556886244943, Tokens per sec: 81962.26241255956, Loss: 2.2631046772003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8393 , TFLOPS: 102.25381762698015, Tokens per sec: 83554.2266304425, Loss: 2.277492046356201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8394 , TFLOPS: 100.94073542763545, Tokens per sec: 82481.27336361504, Loss: 2.2660906314849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8395 , TFLOPS: 101.53375660085835, Tokens per sec: 82965.84622997855, Loss: 2.257448196411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8396 , TFLOPS: 102.74088403458096, Tokens per sec: 83952.22113029787, Loss: 2.289447069168091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8397 , TFLOPS: 102.75907103686984, Tokens per sec: 83967.08219804321, Loss: 2.2693684101104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8398 , TFLOPS: 100.87141869826237, Tokens per sec: 82424.63287967294, Loss: 2.2518250942230225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8399 , TFLOPS: 103.34098901577, Tokens per sec: 84442.58235850396, Loss: 2.2794888019561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8400 , TFLOPS: 101.45680485338198, Tokens per sec: 82902.9669762014, Loss: 2.280488967895508 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/model.pt +[rank0]:[2024-08-28 11:18:13,231] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007633633998921141, 'preprocessing_with_comm': 0.0014769050176255405, 'state_converting': 3.3616077639744617, : 3.3723687370074913}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400, took 15.79s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006400 + eval ppl=7.719850063323975, eval loss=2.043794870376587 +------------------------------------------------------------------ +iteration: 8401 , TFLOPS: 100.4983553173896, Tokens per sec: 82119.79318765603, Loss: 2.279345750808716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8402 , TFLOPS: 99.99275481006015, Tokens per sec: 81706.6539978022, Loss: 2.2871804237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8403 , TFLOPS: 102.40651418059551, Tokens per sec: 83678.99891516058, Loss: 2.280181884765625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8404 , TFLOPS: 101.73932986054463, Tokens per sec: 83133.8254323947, Loss: 2.2596046924591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8405 , TFLOPS: 99.45864312827028, Tokens per sec: 81270.21759335384, Loss: 2.2764015197753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8406 , TFLOPS: 102.29159504023744, Tokens per sec: 83585.09552729216, Loss: 2.2744863033294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8407 , TFLOPS: 102.26513572569013, Tokens per sec: 83563.4749402522, Loss: 2.2801642417907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8408 , TFLOPS: 102.19825789862458, Tokens per sec: 83508.8273461685, Loss: 2.2406797409057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8409 , TFLOPS: 102.67525579920076, Tokens per sec: 83898.59460974764, Loss: 2.2595314979553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8410 , TFLOPS: 103.4152831176035, Tokens per sec: 84503.28998161243, Loss: 2.2445321083068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8411 , TFLOPS: 102.12867378275082, Tokens per sec: 83451.96837383357, Loss: 2.2703421115875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8412 , TFLOPS: 103.36611599058709, Tokens per sec: 84463.11425645289, Loss: 2.2713146209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8413 , TFLOPS: 101.96621025194455, Tokens per sec: 83319.21524062841, Loss: 2.2821156978607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8414 , TFLOPS: 101.7628316846198, Tokens per sec: 83153.02937783729, Loss: 2.2560157775878906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8415 , TFLOPS: 102.75890135676671, Tokens per sec: 83966.94354806299, Loss: 2.264901638031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8416 , TFLOPS: 101.90561420727734, Tokens per sec: 83269.70065264989, Loss: 2.2793807983398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8417 , TFLOPS: 101.31993707775452, Tokens per sec: 82791.12879344622, Loss: 2.2676138877868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8418 , TFLOPS: 103.3619656969782, Tokens per sec: 84459.72294470698, Loss: 2.2507970333099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8419 , TFLOPS: 102.06218205893124, Tokens per sec: 83397.63627465135, Loss: 2.257925033569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8420 , TFLOPS: 102.06952785985256, Tokens per sec: 83403.63871768188, Loss: 2.2586145401000977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8421 , TFLOPS: 101.4247566157235, Tokens per sec: 82876.77953621563, Loss: 2.235224485397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8422 , TFLOPS: 102.15888145210324, Tokens per sec: 83476.65183807602, Loss: 2.2654762268066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8423 , TFLOPS: 102.05402628575689, Tokens per sec: 83390.97198244231, Loss: 2.2769198417663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8424 , TFLOPS: 100.75095845151317, Tokens per sec: 82326.20171113157, Loss: 2.2533886432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8425 , TFLOPS: 102.68620860467415, Tokens per sec: 83907.54442905025, Loss: 2.280651569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8426 , TFLOPS: 101.9030739373078, Tokens per sec: 83267.6249326654, Loss: 2.2653162479400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8427 , TFLOPS: 99.81098441296344, Tokens per sec: 81558.12472714817, Loss: 2.266899347305298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8428 , TFLOPS: 103.2878199275188, Tokens per sec: 84399.13652779981, Loss: 2.2856311798095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8429 , TFLOPS: 101.09770714660968, Tokens per sec: 82609.53899600041, Loss: 2.250917434692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8430 , TFLOPS: 99.85480171722008, Tokens per sec: 81593.92897441404, Loss: 2.2727155685424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8431 , TFLOPS: 102.76075248699259, Tokens per sec: 83968.45615422288, Loss: 2.2662525177001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8432 , TFLOPS: 101.45281501522905, Tokens per sec: 82899.70677673907, Loss: 2.2669098377227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8433 , TFLOPS: 101.44756576248592, Tokens per sec: 82895.41747719487, Loss: 2.2372264862060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8434 , TFLOPS: 101.52712634008374, Tokens per sec: 82960.42847322152, Loss: 2.2725131511688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8435 , TFLOPS: 102.12804062790276, Tokens per sec: 83451.45100670836, Loss: 2.2768211364746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8436 , TFLOPS: 100.9539867134569, Tokens per sec: 82492.1013303782, Loss: 2.265687942504883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8437 , TFLOPS: 101.55677828965634, Tokens per sec: 82984.65784452637, Loss: 2.267770528793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8438 , TFLOPS: 100.87362395967689, Tokens per sec: 82426.43485554322, Loss: 2.227491855621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8439 , TFLOPS: 101.5283572169423, Tokens per sec: 82961.43425438824, Loss: 2.267305850982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8440 , TFLOPS: 99.73027795339132, Tokens per sec: 81492.17740146272, Loss: 2.2713117599487305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8441 , TFLOPS: 102.27491800768246, Tokens per sec: 83571.46829468646, Loss: 2.2752163410186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8442 , TFLOPS: 101.12410009640578, Tokens per sec: 82631.10535469378, Loss: 2.278214931488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8443 , TFLOPS: 100.53210222175173, Tokens per sec: 82147.36865193293, Loss: 2.247915267944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8444 , TFLOPS: 102.77565015020758, Tokens per sec: 83980.62941833583, Loss: 2.2654809951782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8445 , TFLOPS: 102.24293746737393, Tokens per sec: 83545.33617194803, Loss: 2.236034870147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8446 , TFLOPS: 100.64669441109305, Tokens per sec: 82241.0048797091, Loss: 2.2700774669647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8447 , TFLOPS: 103.35079230418133, Tokens per sec: 84450.59287782393, Loss: 2.2641072273254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8448 , TFLOPS: 103.41143012501841, Tokens per sec: 84500.14160218631, Loss: 2.274644613265991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8449 , TFLOPS: 102.15655931467565, Tokens per sec: 83474.7543598068, Loss: 2.255150556564331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8450 , TFLOPS: 103.35378722492538, Tokens per sec: 84453.04010466003, Loss: 2.250493288040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8451 , TFLOPS: 101.40108727418209, Tokens per sec: 82857.43870793909, Loss: 2.268216609954834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8452 , TFLOPS: 102.66423306930108, Tokens per sec: 83889.5876533963, Loss: 2.2422144412994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8453 , TFLOPS: 102.67247580963331, Tokens per sec: 83896.32300870879, Loss: 2.2719342708587646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8454 , TFLOPS: 101.91218116522106, Tokens per sec: 83275.06668303422, Loss: 2.2804863452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8455 , TFLOPS: 101.48121420747319, Tokens per sec: 82922.912487782, Loss: 2.276500940322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8456 , TFLOPS: 102.79298366682654, Tokens per sec: 83994.7930809696, Loss: 2.2784862518310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8457 , TFLOPS: 102.08926399367947, Tokens per sec: 83419.76562068514, Loss: 2.2569944858551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8458 , TFLOPS: 101.82455141973446, Tokens per sec: 83203.4621622068, Loss: 2.2848567962646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8459 , TFLOPS: 101.55916707189485, Tokens per sec: 82986.60977998619, Loss: 2.262138843536377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8460 , TFLOPS: 102.6055010822066, Tokens per sec: 83841.59623484555, Loss: 2.253680944442749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8461 , TFLOPS: 100.98389107126019, Tokens per sec: 82516.53695094591, Loss: 2.2662765979766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8462 , TFLOPS: 101.7637331031687, Tokens per sec: 83153.76595013817, Loss: 2.2570910453796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8463 , TFLOPS: 102.07251675902675, Tokens per sec: 83406.08102413824, Loss: 2.2676172256469727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8464 , TFLOPS: 102.85462602790837, Tokens per sec: 84045.16264102509, Loss: 2.2594997882843018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8465 , TFLOPS: 100.42328122238357, Tokens per sec: 82058.44821204746, Loss: 2.2683026790618896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8466 , TFLOPS: 103.25443846223521, Tokens per sec: 84371.85967320124, Loss: 2.261988401412964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8467 , TFLOPS: 101.7643844840969, Tokens per sec: 83154.29821026264, Loss: 2.260754346847534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8468 , TFLOPS: 100.3160754755682, Tokens per sec: 81970.8476366032, Loss: 2.276770830154419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8469 , TFLOPS: 100.7473366470535, Tokens per sec: 82323.24224147401, Loss: 2.285804271697998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8470 , TFLOPS: 100.7398732936887, Tokens per sec: 82317.14374331583, Loss: 2.264014482498169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8471 , TFLOPS: 100.43410638288204, Tokens per sec: 82067.29372935518, Loss: 2.2697975635528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8472 , TFLOPS: 100.94678796111687, Tokens per sec: 82486.21904452877, Loss: 2.2583513259887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8473 , TFLOPS: 102.19879929302029, Tokens per sec: 83509.26973346598, Loss: 2.276031970977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8474 , TFLOPS: 100.42057304686824, Tokens per sec: 82056.23529211925, Loss: 2.2770893573760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8475 , TFLOPS: 101.67963017930967, Tokens per sec: 83085.04328605109, Loss: 2.2674849033355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8476 , TFLOPS: 101.50768528344865, Tokens per sec: 82944.54268538767, Loss: 2.2700254917144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8477 , TFLOPS: 101.56638337746975, Tokens per sec: 82992.50640903568, Loss: 2.2707371711730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8478 , TFLOPS: 99.8592603997779, Tokens per sec: 81597.57227870873, Loss: 2.2577099800109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8479 , TFLOPS: 102.18810487481076, Tokens per sec: 83500.53104904816, Loss: 2.2799882888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8480 , TFLOPS: 101.64207787710431, Tokens per sec: 83054.35833323684, Loss: 2.276395082473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8481 , TFLOPS: 99.37439253270752, Tokens per sec: 81201.37426291623, Loss: 2.270939350128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8482 , TFLOPS: 102.68709298881674, Tokens per sec: 83908.26708209922, Loss: 2.2623610496520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8483 , TFLOPS: 101.42944024822025, Tokens per sec: 82880.60665289615, Loss: 2.2646820545196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8484 , TFLOPS: 102.28982576377648, Tokens per sec: 83583.64980595067, Loss: 2.254268169403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8485 , TFLOPS: 102.68955803756985, Tokens per sec: 83910.28133689139, Loss: 2.252357244491577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8486 , TFLOPS: 103.3176400851039, Tokens per sec: 84423.50334620175, Loss: 2.2502942085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8487 , TFLOPS: 102.09265575347055, Tokens per sec: 83422.5371149221, Loss: 2.269791603088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8488 , TFLOPS: 102.60058587099769, Tokens per sec: 83837.57988923795, Loss: 2.2810232639312744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8489 , TFLOPS: 101.53742383626795, Tokens per sec: 82968.84282243477, Loss: 2.2513256072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8490 , TFLOPS: 103.39565666478279, Tokens per sec: 84487.25270178278, Loss: 2.285499095916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8491 , TFLOPS: 102.17652860845587, Tokens per sec: 83491.07178380997, Loss: 2.283140182495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8492 , TFLOPS: 101.46585455528998, Tokens per sec: 82910.36172059049, Loss: 2.2545721530914307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8493 , TFLOPS: 101.95618544286636, Tokens per sec: 83311.02371106908, Loss: 2.2491414546966553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8494 , TFLOPS: 102.24917340423411, Tokens per sec: 83550.43171648378, Loss: 2.2567200660705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8495 , TFLOPS: 102.73166003785519, Tokens per sec: 83944.68396512645, Loss: 2.2585506439208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8496 , TFLOPS: 102.63089216162936, Tokens per sec: 83862.34393946643, Loss: 2.260350227355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8497 , TFLOPS: 101.54375180240864, Tokens per sec: 82974.01356646467, Loss: 2.2788305282592773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8498 , TFLOPS: 102.65988235959117, Tokens per sec: 83886.0325764951, Loss: 2.262612819671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8499 , TFLOPS: 101.2422953739401, Tokens per sec: 82727.6857585842, Loss: 2.2622361183166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8500 , TFLOPS: 102.16806592200084, Tokens per sec: 83484.15670485863, Loss: 2.245514392852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8501 , TFLOPS: 102.74156753720645, Tokens per sec: 83952.77963788809, Loss: 2.261803388595581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8502 , TFLOPS: 101.49561725662575, Tokens per sec: 82934.68159001203, Loss: 2.2716636657714844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8503 , TFLOPS: 100.99265569415118, Tokens per sec: 82523.6987499316, Loss: 2.268054485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8504 , TFLOPS: 102.60592996225401, Tokens per sec: 83841.94668377265, Loss: 2.256636619567871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8505 , TFLOPS: 102.14836114490245, Tokens per sec: 83468.05542424545, Loss: 2.258159875869751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8506 , TFLOPS: 100.4340060234784, Tokens per sec: 82067.211723103, Loss: 2.2691638469696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8507 , TFLOPS: 102.08328559650009, Tokens per sec: 83414.88051845224, Loss: 2.2568564414978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8508 , TFLOPS: 101.4323256359174, Tokens per sec: 82882.96437745995, Loss: 2.2656264305114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8509 , TFLOPS: 100.4870041007624, Tokens per sec: 82110.51781634364, Loss: 2.279920816421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8510 , TFLOPS: 99.95629065859761, Tokens per sec: 81676.85820097139, Loss: 2.2889482975006104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8511 , TFLOPS: 102.64297853163339, Tokens per sec: 83872.22002352755, Loss: 2.2765915393829346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8512 , TFLOPS: 99.95408228304967, Tokens per sec: 81675.05368046252, Loss: 2.269049882888794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8513 , TFLOPS: 101.21499185000387, Tokens per sec: 82705.37534632067, Loss: 2.2340312004089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8514 , TFLOPS: 100.94033386556883, Tokens per sec: 82480.94523691328, Loss: 2.2412562370300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8515 , TFLOPS: 101.47171074772652, Tokens per sec: 82915.14696618219, Loss: 2.2488605976104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8516 , TFLOPS: 100.94480112567746, Tokens per sec: 82484.59555014555, Loss: 2.2617146968841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8517 , TFLOPS: 101.5972838905043, Tokens per sec: 83017.75601368617, Loss: 2.25665545463562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8518 , TFLOPS: 102.02942521276613, Tokens per sec: 83370.8698124136, Loss: 2.288062572479248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8519 , TFLOPS: 100.41020034200228, Tokens per sec: 82047.75948795611, Loss: 2.258415699005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8520 , TFLOPS: 102.78045954502203, Tokens per sec: 83984.55929864354, Loss: 2.2781784534454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8521 , TFLOPS: 101.71369400950329, Tokens per sec: 83112.8777185833, Loss: 2.2569003105163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8522 , TFLOPS: 101.48268366084793, Tokens per sec: 82924.11321596158, Loss: 2.244809627532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8523 , TFLOPS: 102.06722350541077, Tokens per sec: 83401.75577034814, Loss: 2.258807897567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8524 , TFLOPS: 102.9189978829483, Tokens per sec: 84097.76254085719, Loss: 2.2768356800079346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8525 , TFLOPS: 102.78791718855727, Tokens per sec: 83990.65313115301, Loss: 2.2631897926330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8526 , TFLOPS: 102.79223342917942, Tokens per sec: 83994.18004247529, Loss: 2.261986494064331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8527 , TFLOPS: 102.03110171046922, Tokens per sec: 83372.23972184368, Loss: 2.2899513244628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8528 , TFLOPS: 103.22935219373512, Tokens per sec: 84351.3610374318, Loss: 2.2721681594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8529 , TFLOPS: 101.95515546258589, Tokens per sec: 83310.18208766787, Loss: 2.260984420776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8530 , TFLOPS: 102.73281309969673, Tokens per sec: 83945.62616163962, Loss: 2.3011972904205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8531 , TFLOPS: 102.09984224871693, Tokens per sec: 83428.40938518476, Loss: 2.257158041000366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8532 , TFLOPS: 102.6955461658649, Tokens per sec: 83915.17439066942, Loss: 2.2781028747558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8533 , TFLOPS: 102.44679369477183, Tokens per sec: 83711.91234308099, Loss: 2.2782058715820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8534 , TFLOPS: 101.93916012790201, Tokens per sec: 83297.11188794112, Loss: 2.2671849727630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8535 , TFLOPS: 101.40837980325217, Tokens per sec: 82863.39762116865, Loss: 2.2774224281311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8536 , TFLOPS: 102.67136363790216, Tokens per sec: 83895.4142245574, Loss: 2.2548184394836426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8537 , TFLOPS: 102.04063620710546, Tokens per sec: 83380.03060448494, Loss: 2.2446465492248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8538 , TFLOPS: 101.54830304400616, Tokens per sec: 82977.73250313339, Loss: 2.2761552333831787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8539 , TFLOPS: 102.65392527190413, Tokens per sec: 83881.16488679695, Loss: 2.279470920562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8540 , TFLOPS: 102.19787305608233, Tokens per sec: 83508.51288142051, Loss: 2.2464652061462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8541 , TFLOPS: 100.28124213162577, Tokens per sec: 81942.3844145778, Loss: 2.2685720920562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8542 , TFLOPS: 103.476300182047, Tokens per sec: 84553.1485956882, Loss: 2.2589187622070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8543 , TFLOPS: 102.32314555465155, Tokens per sec: 83610.87626480241, Loss: 2.2559378147125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8544 , TFLOPS: 99.7141816341655, Tokens per sec: 81479.02468466718, Loss: 2.284956693649292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8545 , TFLOPS: 102.31360724022396, Tokens per sec: 83603.08226254558, Loss: 2.2605068683624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8546 , TFLOPS: 101.47866196361082, Tokens per sec: 82920.82698362146, Loss: 2.2827839851379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8547 , TFLOPS: 101.5613566138257, Tokens per sec: 82988.39891106171, Loss: 2.271000623703003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8548 , TFLOPS: 99.65640172318602, Tokens per sec: 81431.81123201862, Loss: 2.2607502937316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8549 , TFLOPS: 102.67415693500156, Tokens per sec: 83897.69669952319, Loss: 2.284879684448242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8550 , TFLOPS: 99.69447207689943, Tokens per sec: 81462.91949805593, Loss: 2.2869324684143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8551 , TFLOPS: 101.1404492012837, Tokens per sec: 82644.46462915294, Loss: 2.2689156532287598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8552 , TFLOPS: 100.81158249429367, Tokens per sec: 82375.7391770892, Loss: 2.266306161880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8553 , TFLOPS: 101.66708836509383, Tokens per sec: 83074.79504680033, Loss: 2.279559373855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8554 , TFLOPS: 100.51087652647473, Tokens per sec: 82130.0245899241, Loss: 2.27453351020813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8555 , TFLOPS: 101.05496946516648, Tokens per sec: 82574.61693632738, Loss: 2.281812906265259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8556 , TFLOPS: 102.04518238879393, Tokens per sec: 83383.7454065717, Loss: 2.2626583576202393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8557 , TFLOPS: 100.41856928972823, Tokens per sec: 82054.59797057885, Loss: 2.263956069946289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8558 , TFLOPS: 102.74558573585064, Tokens per sec: 83956.06301143789, Loss: 2.2615532875061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8559 , TFLOPS: 101.06143430410702, Tokens per sec: 82579.89952264546, Loss: 2.2719054222106934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8560 , TFLOPS: 102.00366747287988, Tokens per sec: 83349.82250008933, Loss: 2.2679789066314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8561 , TFLOPS: 102.95920272675022, Tokens per sec: 84130.61495369248, Loss: 2.263507604598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8562 , TFLOPS: 102.77568021436745, Tokens per sec: 83980.65398453479, Loss: 2.273049831390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8563 , TFLOPS: 102.7263902801158, Tokens per sec: 83940.37791042196, Loss: 2.2635927200317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8564 , TFLOPS: 101.07377702000481, Tokens per sec: 82589.98507353531, Loss: 2.266448974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8565 , TFLOPS: 102.57181819567155, Tokens per sec: 83814.07308118313, Loss: 2.2659952640533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8566 , TFLOPS: 103.34173118248454, Tokens per sec: 84443.18880203148, Loss: 2.243027448654175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8567 , TFLOPS: 102.1703610166286, Tokens per sec: 83486.03208575997, Loss: 2.256997585296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8568 , TFLOPS: 102.13648743956337, Tokens per sec: 83458.35311395652, Loss: 2.2528326511383057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8569 , TFLOPS: 102.09222085160896, Tokens per sec: 83422.18174541563, Loss: 2.2643110752105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8570 , TFLOPS: 102.67503644326916, Tokens per sec: 83898.41536836921, Loss: 2.27427339553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8571 , TFLOPS: 101.81254636436454, Tokens per sec: 83193.65252242642, Loss: 2.255134344100952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8572 , TFLOPS: 102.09908762979906, Tokens per sec: 83427.79276664135, Loss: 2.2655630111694336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8573 , TFLOPS: 102.133303641543, Tokens per sec: 83455.75155063583, Loss: 2.2463274002075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8574 , TFLOPS: 102.65892611215723, Tokens per sec: 83885.25120210083, Loss: 2.253432273864746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8575 , TFLOPS: 101.50477588127104, Tokens per sec: 82942.16533797367, Loss: 2.28261661529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8576 , TFLOPS: 102.21577072919166, Tokens per sec: 83523.13753084524, Loss: 2.268455743789673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8577 , TFLOPS: 102.20349745187622, Tokens per sec: 83513.10872000853, Loss: 2.263392925262451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8578 , TFLOPS: 102.10894678016747, Tokens per sec: 83435.84893220449, Loss: 2.27756667137146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8579 , TFLOPS: 101.03925885600901, Tokens per sec: 82561.77939315776, Loss: 2.2778449058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8580 , TFLOPS: 102.18510137287757, Tokens per sec: 83498.07681030153, Loss: 2.2753257751464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8581 , TFLOPS: 102.74332124650684, Tokens per sec: 83954.21263890192, Loss: 2.265216588973999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8582 , TFLOPS: 99.9569507242702, Tokens per sec: 81677.39755762408, Loss: 2.2638697624206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8583 , TFLOPS: 102.08125128703105, Tokens per sec: 83413.2182318173, Loss: 2.2801849842071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8584 , TFLOPS: 101.3131995371926, Tokens per sec: 82785.62337561334, Loss: 2.260918140411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8585 , TFLOPS: 101.23391194780052, Tokens per sec: 82720.83544527696, Loss: 2.240269660949707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8586 , TFLOPS: 101.64853955967124, Tokens per sec: 83059.63834040078, Loss: 2.2560741901397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8587 , TFLOPS: 101.4920570841142, Tokens per sec: 82931.77248140583, Loss: 2.275285005569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8588 , TFLOPS: 100.93441932992087, Tokens per sec: 82476.11231758163, Loss: 2.289011001586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8589 , TFLOPS: 101.66652483139349, Tokens per sec: 83074.33456890713, Loss: 2.270646810531616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8590 , TFLOPS: 100.85970585184006, Tokens per sec: 82415.06201134577, Loss: 2.283294439315796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8591 , TFLOPS: 101.81618708064565, Tokens per sec: 83196.62744542009, Loss: 2.2800774574279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8592 , TFLOPS: 99.59013686303271, Tokens per sec: 81377.66450897808, Loss: 2.2701120376586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8593 , TFLOPS: 101.02060950239405, Tokens per sec: 82546.54052624191, Loss: 2.2786974906921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8594 , TFLOPS: 101.66045622457254, Tokens per sec: 83069.37575405388, Loss: 2.276689052581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8595 , TFLOPS: 100.57612870964873, Tokens per sec: 82183.34382853618, Loss: 2.2715485095977783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8596 , TFLOPS: 102.27196127373969, Tokens per sec: 83569.05227126874, Loss: 2.23563551902771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8597 , TFLOPS: 100.94638699183089, Tokens per sec: 82485.89140220334, Loss: 2.258335828781128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8598 , TFLOPS: 102.27631845600838, Tokens per sec: 83572.61263706433, Loss: 2.27217173576355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8599 , TFLOPS: 103.47888396845904, Tokens per sec: 84555.25987407805, Loss: 2.2627151012420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8600 , TFLOPS: 103.38740045687219, Tokens per sec: 84480.50634176533, Loss: 2.2653920650482178 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/model.pt +[rank0]:[2024-08-28 12:11:09,088] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007717749016592279, 'preprocessing_with_comm': 0.0015106670034583658, 'state_converting': 3.3619066090031993, : 3.37276950199157}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600, took 15.75s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006600 + eval ppl=7.606335163116455, eval loss=2.0289814472198486 +------------------------------------------------------------------ +iteration: 8601 , TFLOPS: 100.64381876750053, Tokens per sec: 82238.65511730399, Loss: 2.2502543926239014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8602 , TFLOPS: 99.34797378236361, Tokens per sec: 81179.78682193109, Loss: 2.2386600971221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8603 , TFLOPS: 101.51664102296645, Tokens per sec: 82951.8606506889, Loss: 2.2769784927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8604 , TFLOPS: 100.23459304427031, Tokens per sec: 81904.26624444532, Loss: 2.2836456298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8605 , TFLOPS: 99.88261932907221, Tokens per sec: 81616.65946114746, Loss: 2.2726504802703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8606 , TFLOPS: 102.38311471694483, Tokens per sec: 83659.87861105584, Loss: 2.267061710357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8607 , TFLOPS: 101.11646828526953, Tokens per sec: 82624.86920535404, Loss: 2.2801475524902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8608 , TFLOPS: 100.74027982332387, Tokens per sec: 82317.47592914573, Loss: 2.243297576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8609 , TFLOPS: 102.419197735657, Tokens per sec: 83689.36297450507, Loss: 2.2596232891082764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8610 , TFLOPS: 102.6528493223374, Tokens per sec: 83880.28570070855, Loss: 2.267153263092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8611 , TFLOPS: 103.27053240445385, Tokens per sec: 84385.01044768299, Loss: 2.2463061809539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8612 , TFLOPS: 102.00811334428893, Tokens per sec: 83353.45533606458, Loss: 2.2530643939971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8613 , TFLOPS: 103.41030343197576, Tokens per sec: 84499.2209522975, Loss: 2.2559280395507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8614 , TFLOPS: 101.7959904023473, Tokens per sec: 83180.12421967377, Loss: 2.270695209503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8615 , TFLOPS: 103.21773770682539, Tokens per sec: 84341.87054119437, Loss: 2.2497918605804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8616 , TFLOPS: 101.36270100957498, Tokens per sec: 82826.07230297825, Loss: 2.2895212173461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8617 , TFLOPS: 101.9701828082437, Tokens per sec: 83322.46131864315, Loss: 2.2790210247039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8618 , TFLOPS: 102.71073547242536, Tokens per sec: 83927.58596406749, Loss: 2.265333414077759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8619 , TFLOPS: 102.57541357509889, Tokens per sec: 83817.0109582665, Loss: 2.2723302841186523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8620 , TFLOPS: 101.3342249457684, Tokens per sec: 82802.80377820281, Loss: 2.2675886154174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8621 , TFLOPS: 103.3187089804972, Tokens per sec: 84424.37676814343, Loss: 2.243912696838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8622 , TFLOPS: 102.1354775596906, Tokens per sec: 83457.52791511586, Loss: 2.2532267570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8623 , TFLOPS: 101.8924111925582, Tokens per sec: 83258.91212944704, Loss: 2.2559430599212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8624 , TFLOPS: 100.6802942360731, Tokens per sec: 82268.46016163673, Loss: 2.2721657752990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8625 , TFLOPS: 103.37648458671111, Tokens per sec: 84471.58670325723, Loss: 2.2712552547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8626 , TFLOPS: 102.17689531198809, Tokens per sec: 83491.37142670593, Loss: 2.2664804458618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8627 , TFLOPS: 100.42488690460766, Tokens per sec: 82059.7602563267, Loss: 2.272754669189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8628 , TFLOPS: 103.3136791403546, Tokens per sec: 84420.2667562833, Loss: 2.262441635131836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8629 , TFLOPS: 100.13363494391615, Tokens per sec: 81821.7707817532, Loss: 2.275009870529175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8630 , TFLOPS: 101.69208959509388, Tokens per sec: 83095.22419542226, Loss: 2.2561235427856445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8631 , TFLOPS: 101.61717950928401, Tokens per sec: 83034.01323594987, Loss: 2.259181022644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8632 , TFLOPS: 102.86966303101127, Tokens per sec: 84057.44976334737, Loss: 2.2683494091033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8633 , TFLOPS: 99.76098810265287, Tokens per sec: 81517.27145497389, Loss: 2.2574992179870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8634 , TFLOPS: 102.08202734532098, Tokens per sec: 83413.85236902352, Loss: 2.2617735862731934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8635 , TFLOPS: 99.87431505116602, Tokens per sec: 81609.87382189918, Loss: 2.2792346477508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8636 , TFLOPS: 102.76675382175162, Tokens per sec: 83973.35999934259, Loss: 2.2595832347869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8637 , TFLOPS: 99.61228486542193, Tokens per sec: 81395.76221187062, Loss: 2.2647345066070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8638 , TFLOPS: 101.28297362810129, Tokens per sec: 82760.92500721071, Loss: 2.261549949645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8639 , TFLOPS: 101.77471917458037, Tokens per sec: 83162.74295189528, Loss: 2.289227247238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8640 , TFLOPS: 99.35697740413698, Tokens per sec: 81187.14391305596, Loss: 2.2623603343963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8641 , TFLOPS: 101.94446573184567, Tokens per sec: 83301.44723350188, Loss: 2.2687554359436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8642 , TFLOPS: 101.82217892283751, Tokens per sec: 83201.52353391881, Loss: 2.2834620475769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8643 , TFLOPS: 100.94803904701288, Tokens per sec: 82487.24133902023, Loss: 2.271519184112549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8644 , TFLOPS: 103.32466118020437, Tokens per sec: 84429.2404637474, Loss: 2.2711851596832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8645 , TFLOPS: 102.35961521959509, Tokens per sec: 83640.67656684048, Loss: 2.255892515182495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8646 , TFLOPS: 101.58458746285176, Tokens per sec: 83007.38143581615, Loss: 2.2695271968841553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8647 , TFLOPS: 103.27141806804815, Tokens per sec: 84385.73414620491, Loss: 2.271430015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8648 , TFLOPS: 100.54754647494818, Tokens per sec: 82159.98854879019, Loss: 2.2659542560577393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8649 , TFLOPS: 103.2705782218497, Tokens per sec: 84385.0478862566, Loss: 2.2618565559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8650 , TFLOPS: 102.07325386718713, Tokens per sec: 83406.68333419094, Loss: 2.265836477279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8651 , TFLOPS: 103.27859386588003, Tokens per sec: 84391.59767533481, Loss: 2.2622807025909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8652 , TFLOPS: 100.71911882628639, Tokens per sec: 82300.18473373388, Loss: 2.260627508163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8653 , TFLOPS: 103.37101811581071, Tokens per sec: 84467.11990916503, Loss: 2.2781269550323486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8654 , TFLOPS: 101.43496594858603, Tokens per sec: 82885.1218449092, Loss: 2.272446870803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8655 , TFLOPS: 102.12593826495886, Tokens per sec: 83449.73311182727, Loss: 2.263455390930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8656 , TFLOPS: 101.48301915392165, Tokens per sec: 82924.3873559885, Loss: 2.261889934539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8657 , TFLOPS: 102.00374156077784, Tokens per sec: 83349.88303921794, Loss: 2.285900115966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8658 , TFLOPS: 102.04605486419536, Tokens per sec: 83384.45832868172, Loss: 2.2603602409362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8659 , TFLOPS: 102.19541798738324, Tokens per sec: 83506.5067815874, Loss: 2.274535655975342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8660 , TFLOPS: 101.97128866129907, Tokens per sec: 83323.36493964189, Loss: 2.2587344646453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8661 , TFLOPS: 102.21808465907401, Tokens per sec: 83525.02830251797, Loss: 2.2302632331848145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8662 , TFLOPS: 101.68352995694113, Tokens per sec: 83088.22989474297, Loss: 2.2696938514709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8663 , TFLOPS: 102.13836990023427, Tokens per sec: 83459.89132102956, Loss: 2.247978448867798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8664 , TFLOPS: 102.02576488737114, Tokens per sec: 83367.8788663082, Loss: 2.2444844245910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8665 , TFLOPS: 99.8095119909136, Tokens per sec: 81556.92157318775, Loss: 2.2880401611328125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8666 , TFLOPS: 103.36420443176625, Tokens per sec: 84461.55227253247, Loss: 2.306936502456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8667 , TFLOPS: 100.87550016567901, Tokens per sec: 82427.9679517654, Loss: 2.2731270790100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8668 , TFLOPS: 101.40573486083153, Tokens per sec: 82861.23637062986, Loss: 2.278210163116455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8669 , TFLOPS: 101.58671912384004, Tokens per sec: 83009.12327088352, Loss: 2.265160322189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8670 , TFLOPS: 100.16045033300621, Tokens per sec: 81843.68232646823, Loss: 2.284585952758789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8671 , TFLOPS: 101.77565220173297, Tokens per sec: 83163.50535239992, Loss: 2.2529821395874023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8672 , TFLOPS: 101.13758065008298, Tokens per sec: 82642.12066212374, Loss: 2.2959213256835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8673 , TFLOPS: 100.90636397893927, Tokens per sec: 82453.18756808559, Loss: 2.273571252822876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8674 , TFLOPS: 102.72859969389577, Tokens per sec: 83942.18327929695, Loss: 2.2750051021575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8675 , TFLOPS: 100.17757987643672, Tokens per sec: 81857.67931735897, Loss: 2.254561185836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8676 , TFLOPS: 102.86729005521724, Tokens per sec: 84055.51074374023, Loss: 2.2622625827789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8677 , TFLOPS: 101.18085322777505, Tokens per sec: 82677.47979929118, Loss: 2.2691729068756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8678 , TFLOPS: 100.24462914598342, Tokens per sec: 81912.46700150732, Loss: 2.2795093059539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8679 , TFLOPS: 102.04396684848624, Tokens per sec: 83382.7521572955, Loss: 2.26545786857605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8680 , TFLOPS: 101.00911801073191, Tokens per sec: 82537.15053258745, Loss: 2.266214609146118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8681 , TFLOPS: 102.23449480180933, Tokens per sec: 83538.43745258165, Loss: 2.2612314224243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8682 , TFLOPS: 102.63789620223518, Tokens per sec: 83868.06712134549, Loss: 2.265183687210083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8683 , TFLOPS: 102.82468508421502, Tokens per sec: 84020.6971251848, Loss: 2.2770233154296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8684 , TFLOPS: 102.23770755345934, Tokens per sec: 83541.06267465846, Loss: 2.2756261825561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8685 , TFLOPS: 102.7024065877677, Tokens per sec: 83920.78021800917, Loss: 2.2734618186950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8686 , TFLOPS: 101.27016305978768, Tokens per sec: 82750.45716206831, Loss: 2.2728095054626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8687 , TFLOPS: 102.7083385561876, Tokens per sec: 83925.62738210752, Loss: 2.2495288848876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8688 , TFLOPS: 102.07359943461253, Tokens per sec: 83406.96570622992, Loss: 2.269605875015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8689 , TFLOPS: 103.32469381389512, Tokens per sec: 84429.26712957618, Loss: 2.2719521522521973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8690 , TFLOPS: 100.54248819183947, Tokens per sec: 82155.85529545027, Loss: 2.2616169452667236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8691 , TFLOPS: 102.16845047537925, Tokens per sec: 83484.4709333234, Loss: 2.263338565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8692 , TFLOPS: 102.62492360701104, Tokens per sec: 83857.46687984355, Loss: 2.2720913887023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8693 , TFLOPS: 101.020184283604, Tokens per sec: 82546.19306902244, Loss: 2.2912940979003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8694 , TFLOPS: 100.87290808713438, Tokens per sec: 82425.84989766055, Loss: 2.2558603286743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8695 , TFLOPS: 101.63298814578069, Tokens per sec: 83046.9308797818, Loss: 2.2469675540924072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8696 , TFLOPS: 102.6833658799495, Tokens per sec: 83905.2215654995, Loss: 2.246817111968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8697 , TFLOPS: 100.93680995253537, Tokens per sec: 82478.06575686004, Loss: 2.2645318508148193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8698 , TFLOPS: 102.07369157271084, Tokens per sec: 83407.04099464191, Loss: 2.261016607284546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8699 , TFLOPS: 101.53506996544616, Tokens per sec: 82966.91941399226, Loss: 2.2832884788513184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8700 , TFLOPS: 102.0973799229045, Tokens per sec: 83426.39735537751, Loss: 2.276750087738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8701 , TFLOPS: 102.1540100345013, Tokens per sec: 83472.67127735203, Loss: 2.2621912956237793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8702 , TFLOPS: 101.21927645155132, Tokens per sec: 82708.87640454054, Loss: 2.2778637409210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8703 , TFLOPS: 101.60741558604668, Tokens per sec: 83026.0348829269, Loss: 2.2525923252105713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8704 , TFLOPS: 101.55281033611651, Tokens per sec: 82981.41552754476, Loss: 2.271235466003418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8705 , TFLOPS: 101.61672850664344, Tokens per sec: 83033.64471008242, Loss: 2.2687220573425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8706 , TFLOPS: 100.73374968717297, Tokens per sec: 82312.13998679617, Loss: 2.2544915676116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8707 , TFLOPS: 100.82391767515718, Tokens per sec: 82385.81857090857, Loss: 2.272735118865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8708 , TFLOPS: 102.95727474354472, Tokens per sec: 84129.03954898458, Loss: 2.3023171424865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8709 , TFLOPS: 100.70687961366636, Tokens per sec: 82290.18375803668, Loss: 2.2684569358825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8710 , TFLOPS: 101.71401251329735, Tokens per sec: 83113.13797623244, Loss: 2.2649497985839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8711 , TFLOPS: 101.64489193709763, Tokens per sec: 83056.65777409772, Loss: 2.2450051307678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8712 , TFLOPS: 101.91511346509716, Tokens per sec: 83277.4627407471, Loss: 2.2744691371917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8713 , TFLOPS: 102.09281010164923, Tokens per sec: 83422.66323679218, Loss: 2.256438732147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8714 , TFLOPS: 102.51943883466707, Tokens per sec: 83771.27255694165, Loss: 2.2545971870422363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8715 , TFLOPS: 102.22508568594071, Tokens per sec: 83530.7490217932, Loss: 2.279973030090332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8716 , TFLOPS: 101.7328222224576, Tokens per sec: 83128.50787379227, Loss: 2.2605319023132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8717 , TFLOPS: 100.40238031834458, Tokens per sec: 82041.36954532008, Loss: 2.254856824874878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8718 , TFLOPS: 101.03550839107706, Tokens per sec: 82558.7147917163, Loss: 2.260387420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8719 , TFLOPS: 101.58287340780257, Tokens per sec: 83005.98083731177, Loss: 2.2678909301757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8720 , TFLOPS: 102.6747491397717, Tokens per sec: 83898.18060528558, Loss: 2.291750431060791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8721 , TFLOPS: 102.94141395018885, Tokens per sec: 84116.07928644035, Loss: 2.275296211242676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8722 , TFLOPS: 102.14785874292491, Tokens per sec: 83467.64489865662, Loss: 2.2477216720581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8723 , TFLOPS: 103.37489992458492, Tokens per sec: 84470.29183504108, Loss: 2.258223533630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8724 , TFLOPS: 101.08336989401889, Tokens per sec: 82597.82365783476, Loss: 2.277413845062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8725 , TFLOPS: 103.51036714438851, Tokens per sec: 84580.98558757907, Loss: 2.2847800254821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8726 , TFLOPS: 101.13155973631105, Tokens per sec: 82637.20081848859, Loss: 2.2620222568511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8727 , TFLOPS: 103.45156690603206, Tokens per sec: 84532.93839916521, Loss: 2.2694666385650635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8728 , TFLOPS: 102.05098430981433, Tokens per sec: 83388.4863055922, Loss: 2.2512779235839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8729 , TFLOPS: 101.59150718403495, Tokens per sec: 83013.03571812442, Loss: 2.2811787128448486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8730 , TFLOPS: 103.26429116469212, Tokens per sec: 84379.91057000989, Loss: 2.2602450847625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8731 , TFLOPS: 101.44720110248494, Tokens per sec: 82895.11950412088, Loss: 2.2863729000091553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8732 , TFLOPS: 101.51968084921508, Tokens per sec: 82954.34457096875, Loss: 2.2645764350891113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8733 , TFLOPS: 101.33464688374414, Tokens per sec: 82803.14855458442, Loss: 2.2728431224823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8734 , TFLOPS: 103.3703952048933, Tokens per sec: 84466.61091261926, Loss: 2.268321990966797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8735 , TFLOPS: 100.82088698417463, Tokens per sec: 82383.34211528981, Loss: 2.2588870525360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8736 , TFLOPS: 101.96535660498756, Tokens per sec: 83318.51770372504, Loss: 2.2447409629821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8737 , TFLOPS: 100.88106543889103, Tokens per sec: 82432.5154797698, Loss: 2.262361764907837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8738 , TFLOPS: 101.51759096757205, Tokens per sec: 82952.6368748799, Loss: 2.259425401687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8739 , TFLOPS: 102.80398890195075, Tokens per sec: 84003.78574188953, Loss: 2.2427423000335693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8740 , TFLOPS: 100.77516647354281, Tokens per sec: 82345.9827091019, Loss: 2.2683582305908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8741 , TFLOPS: 102.19315414500646, Tokens per sec: 83504.65693770484, Loss: 2.263852119445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8742 , TFLOPS: 101.52796351672211, Tokens per sec: 82961.11255180357, Loss: 2.2862844467163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8743 , TFLOPS: 102.30391778924198, Tokens per sec: 83595.16476271955, Loss: 2.266850471496582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8744 , TFLOPS: 101.03207971824621, Tokens per sec: 82555.91313488431, Loss: 2.252530574798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8745 , TFLOPS: 101.54710672354997, Tokens per sec: 82976.7549588926, Loss: 2.2676594257354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8746 , TFLOPS: 103.45986767824598, Tokens per sec: 84539.72117382211, Loss: 2.2599711418151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8747 , TFLOPS: 99.81898875857837, Tokens per sec: 81564.66528400032, Loss: 2.2738492488861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8748 , TFLOPS: 102.34155019444746, Tokens per sec: 83625.91516975724, Loss: 2.279268264770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8749 , TFLOPS: 101.71802989483696, Tokens per sec: 83116.42068210506, Loss: 2.2712063789367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8750 , TFLOPS: 101.87682056818572, Tokens per sec: 83246.17262893391, Loss: 2.254762649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8751 , TFLOPS: 101.73236849761496, Tokens per sec: 83128.13712354343, Loss: 2.256927013397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8752 , TFLOPS: 102.8969183486665, Tokens per sec: 84079.72078501775, Loss: 2.2801225185394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8753 , TFLOPS: 101.46779845422327, Tokens per sec: 82911.95013044929, Loss: 2.2590255737304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8754 , TFLOPS: 101.20291113564842, Tokens per sec: 82695.50388363558, Loss: 2.2615158557891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8755 , TFLOPS: 102.68958042000583, Tokens per sec: 83910.299626156, Loss: 2.2598443031311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8756 , TFLOPS: 100.42246071362935, Tokens per sec: 82057.77775322262, Loss: 2.2669575214385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8757 , TFLOPS: 102.27164513541429, Tokens per sec: 83568.79394650493, Loss: 2.2696292400360107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8758 , TFLOPS: 101.95295423009144, Tokens per sec: 83308.38340393188, Loss: 2.263458490371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8759 , TFLOPS: 102.79722910277184, Tokens per sec: 83998.26213596773, Loss: 2.2651009559631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8760 , TFLOPS: 102.30750211039347, Tokens per sec: 83598.09360380107, Loss: 2.2562854290008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8761 , TFLOPS: 101.4971119378692, Tokens per sec: 82935.90293253257, Loss: 2.264082431793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8762 , TFLOPS: 102.73443558862687, Tokens per sec: 83946.95193911102, Loss: 2.260697841644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8763 , TFLOPS: 102.7638854247645, Tokens per sec: 83971.0161583253, Loss: 2.2771668434143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8764 , TFLOPS: 100.98875556981184, Tokens per sec: 82520.51185793581, Loss: 2.253061294555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8765 , TFLOPS: 102.04556527814344, Tokens per sec: 83384.05827531563, Loss: 2.2512083053588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8766 , TFLOPS: 102.69393242563422, Tokens per sec: 83913.85576198556, Loss: 2.2701120376586914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8767 , TFLOPS: 100.9690638402709, Tokens per sec: 82504.4212388177, Loss: 2.247392177581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8768 , TFLOPS: 102.03646074970278, Tokens per sec: 83376.61873075538, Loss: 2.2641143798828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8769 , TFLOPS: 100.95275058028918, Tokens per sec: 82491.09125414595, Loss: 2.256283760070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8770 , TFLOPS: 101.91990451915305, Tokens per sec: 83281.37763434889, Loss: 2.2582077980041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8771 , TFLOPS: 101.1327729819955, Tokens per sec: 82638.19219276909, Loss: 2.253089189529419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8772 , TFLOPS: 101.65014780796294, Tokens per sec: 83060.95248147992, Loss: 2.279386043548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8773 , TFLOPS: 102.10213077318842, Tokens per sec: 83430.27939744228, Loss: 2.2566440105438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8774 , TFLOPS: 100.37547881598857, Tokens per sec: 82019.3876352387, Loss: 2.263038158416748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8775 , TFLOPS: 101.4696838173192, Tokens per sec: 82913.49070916852, Loss: 2.2688231468200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8776 , TFLOPS: 101.28390459914502, Tokens per sec: 82761.68572761587, Loss: 2.2688841819763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8777 , TFLOPS: 101.01021970172171, Tokens per sec: 82538.05075265527, Loss: 2.2637362480163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8778 , TFLOPS: 102.43451250673229, Tokens per sec: 83701.87706818798, Loss: 2.2812771797180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8779 , TFLOPS: 100.8635374434714, Tokens per sec: 82418.1929035017, Loss: 2.273031234741211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8780 , TFLOPS: 101.09692670769009, Tokens per sec: 82608.90127926893, Loss: 2.269786834716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8781 , TFLOPS: 102.9243474135217, Tokens per sec: 84102.13377999788, Loss: 2.253694772720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8782 , TFLOPS: 101.57670102927052, Tokens per sec: 83000.93722792207, Loss: 2.2535128593444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8783 , TFLOPS: 102.75759572366178, Tokens per sec: 83965.87668164297, Loss: 2.2681140899658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8784 , TFLOPS: 103.35353856187119, Tokens per sec: 84452.83691567728, Loss: 2.2818479537963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8785 , TFLOPS: 101.51072461819655, Tokens per sec: 82947.0262040496, Loss: 2.262213706970215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8786 , TFLOPS: 102.39933733474146, Tokens per sec: 83673.13452966479, Loss: 2.2979650497436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8787 , TFLOPS: 102.04731036096682, Tokens per sec: 83385.48422741305, Loss: 2.2735836505889893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8788 , TFLOPS: 101.49475380229867, Tokens per sec: 82933.97603925408, Loss: 2.2831568717956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8789 , TFLOPS: 101.91183312596354, Tokens per sec: 83274.7822911976, Loss: 2.2652981281280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8790 , TFLOPS: 101.90664870394066, Tokens per sec: 83270.54596650379, Loss: 2.2657392024993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8791 , TFLOPS: 102.77004656713989, Tokens per sec: 83976.05058636215, Loss: 2.2581515312194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8792 , TFLOPS: 102.22974841894748, Tokens per sec: 83534.55906096246, Loss: 2.2600717544555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8793 , TFLOPS: 101.55590838586102, Tokens per sec: 82983.94702374192, Loss: 2.255431652069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8794 , TFLOPS: 101.63142885499678, Tokens per sec: 83045.65674314256, Loss: 2.2744131088256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8795 , TFLOPS: 100.91688094776973, Tokens per sec: 82461.78125404782, Loss: 2.282064437866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8796 , TFLOPS: 102.1680573797163, Tokens per sec: 83484.14972473803, Loss: 2.2703380584716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8797 , TFLOPS: 102.12673132041168, Tokens per sec: 83450.38113785283, Loss: 2.2745795249938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8798 , TFLOPS: 102.29502647004603, Tokens per sec: 83587.89943692161, Loss: 2.274338483810425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8799 , TFLOPS: 101.79738616185121, Tokens per sec: 83181.26473069446, Loss: 2.2554373741149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8800 , TFLOPS: 102.73776546922154, Tokens per sec: 83949.67287026368, Loss: 2.2726192474365234 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/model.pt +[rank0]:[2024-08-28 13:04:03,772] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007694438012549654, 'preprocessing_with_comm': 0.0014937189989723265, 'state_converting': 3.3305776950146537, : 3.341419358999701}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800, took 15.71s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0006800 + eval ppl=7.4157867431640625, eval loss=2.0036110877990723 +------------------------------------------------------------------ +iteration: 8801 , TFLOPS: 100.88833889103056, Tokens per sec: 82438.45880474834, Loss: 2.2412681579589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8802 , TFLOPS: 100.70315461016067, Tokens per sec: 82287.13996178207, Loss: 2.263047218322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8803 , TFLOPS: 101.72898157589395, Tokens per sec: 83125.36958261809, Loss: 2.2621679306030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8804 , TFLOPS: 100.78789498230452, Tokens per sec: 82356.38350127207, Loss: 2.27199387550354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8805 , TFLOPS: 101.03543284200481, Tokens per sec: 82558.65305862455, Loss: 2.2831363677978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8806 , TFLOPS: 102.27446142171432, Tokens per sec: 83571.09520653835, Loss: 2.2994797229766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8807 , TFLOPS: 101.479035777534, Tokens per sec: 82921.1324366009, Loss: 2.2703518867492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8808 , TFLOPS: 102.2483203587643, Tokens per sec: 83549.73467107106, Loss: 2.2583048343658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8809 , TFLOPS: 103.30347032235827, Tokens per sec: 84411.92486830014, Loss: 2.2879624366760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8810 , TFLOPS: 101.92712126609304, Tokens per sec: 83287.27462404914, Loss: 2.252990484237671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8811 , TFLOPS: 102.0091664492116, Tokens per sec: 83354.31585520624, Loss: 2.2780866622924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8812 , TFLOPS: 102.7234845068374, Tokens per sec: 83938.00352827493, Loss: 2.251349449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8813 , TFLOPS: 102.13264291015484, Tokens per sec: 83455.2116500098, Loss: 2.2588305473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8814 , TFLOPS: 102.62724288121869, Tokens per sec: 83859.36201850204, Loss: 2.273010492324829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8815 , TFLOPS: 102.67554734067389, Tokens per sec: 83898.83283579028, Loss: 2.263977527618408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8816 , TFLOPS: 102.45863445080921, Tokens per sec: 83721.58772964732, Loss: 2.2652182579040527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8817 , TFLOPS: 100.82507044403651, Tokens per sec: 82386.76052803479, Loss: 2.2692925930023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8818 , TFLOPS: 101.68639677595681, Tokens per sec: 83090.57244635918, Loss: 2.263434410095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8819 , TFLOPS: 103.40677912460085, Tokens per sec: 84496.3411500177, Loss: 2.278768539428711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8820 , TFLOPS: 103.4009928295882, Tokens per sec: 84491.61301941039, Loss: 2.241814136505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8821 , TFLOPS: 100.90329705546962, Tokens per sec: 82450.68150596891, Loss: 2.2578072547912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8822 , TFLOPS: 101.30309009350222, Tokens per sec: 82777.3626889337, Loss: 2.2673721313476562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8823 , TFLOPS: 102.66284348203568, Tokens per sec: 83888.45218587058, Loss: 2.275709629058838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8824 , TFLOPS: 102.22391099649192, Tokens per sec: 83529.7891528054, Loss: 2.2756357192993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8825 , TFLOPS: 102.21351663658953, Tokens per sec: 83521.29565375454, Loss: 2.285930871963501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8826 , TFLOPS: 101.98642436551253, Tokens per sec: 83335.73271318241, Loss: 2.273603677749634 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8827 , TFLOPS: 100.2702533948505, Tokens per sec: 81933.40523488345, Loss: 2.2693381309509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8828 , TFLOPS: 102.70480439651624, Tokens per sec: 83922.73952926268, Loss: 2.2788052558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8829 , TFLOPS: 102.16067297572003, Tokens per sec: 83478.11573814122, Loss: 2.289324998855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8830 , TFLOPS: 100.3858252805906, Tokens per sec: 82027.84199780671, Loss: 2.2776355743408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8831 , TFLOPS: 101.57756190063355, Tokens per sec: 83001.6406680736, Loss: 2.2629737854003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8832 , TFLOPS: 101.77192401499501, Tokens per sec: 83160.45895504429, Loss: 2.2368030548095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8833 , TFLOPS: 101.25425448605156, Tokens per sec: 82737.4578569454, Loss: 2.2658371925354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8834 , TFLOPS: 99.67855449046299, Tokens per sec: 81449.91282842359, Loss: 2.2639811038970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8835 , TFLOPS: 102.59431733601967, Tokens per sec: 83832.45770794098, Loss: 2.276085615158081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8836 , TFLOPS: 101.01352256201591, Tokens per sec: 82540.74960482484, Loss: 2.2563717365264893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8837 , TFLOPS: 100.27559277908233, Tokens per sec: 81937.76818318728, Loss: 2.2621209621429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8838 , TFLOPS: 102.19567397506661, Tokens per sec: 83506.7159557132, Loss: 2.258988857269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8839 , TFLOPS: 100.66715369081003, Tokens per sec: 82257.72268383455, Loss: 2.268008232116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8840 , TFLOPS: 102.30281829396544, Tokens per sec: 83594.26633682557, Loss: 2.277615547180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8841 , TFLOPS: 100.86391856271338, Tokens per sec: 82418.50432584519, Loss: 2.259343147277832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8842 , TFLOPS: 101.99218415528232, Tokens per sec: 83340.43918567346, Loss: 2.24605655670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8843 , TFLOPS: 100.9798008535428, Tokens per sec: 82513.19472875746, Loss: 2.2912726402282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8844 , TFLOPS: 102.93800241285781, Tokens per sec: 84113.29163146637, Loss: 2.272831678390503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8845 , TFLOPS: 100.54611469396116, Tokens per sec: 82158.81860368837, Loss: 2.2695536613464355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8846 , TFLOPS: 102.24277736527102, Tokens per sec: 83545.20534839835, Loss: 2.2502105236053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8847 , TFLOPS: 102.04907813271923, Tokens per sec: 83386.92871921844, Loss: 2.2541022300720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8848 , TFLOPS: 102.11964371093336, Tokens per sec: 83444.58966969678, Loss: 2.2597358226776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8849 , TFLOPS: 101.44294445735956, Tokens per sec: 82891.64128981315, Loss: 2.2868595123291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8850 , TFLOPS: 102.26080081938724, Tokens per sec: 83559.9327767217, Loss: 2.2583420276641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8851 , TFLOPS: 103.36683500119781, Tokens per sec: 84463.70177853187, Loss: 2.256404399871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8852 , TFLOPS: 102.71419641498221, Tokens per sec: 83930.4139893233, Loss: 2.2674784660339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8853 , TFLOPS: 102.76759623454451, Tokens per sec: 83974.04835652129, Loss: 2.2551019191741943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8854 , TFLOPS: 102.51166739861756, Tokens per sec: 83764.92231649105, Loss: 2.2865166664123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8855 , TFLOPS: 101.35094783165631, Tokens per sec: 82816.46847874719, Loss: 2.238831043243408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8856 , TFLOPS: 102.23145246577707, Tokens per sec: 83535.95148149299, Loss: 2.27103590965271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8857 , TFLOPS: 103.27230863566017, Tokens per sec: 84386.46185192594, Loss: 2.286557197570801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8858 , TFLOPS: 101.29885516421305, Tokens per sec: 82773.90221919476, Loss: 2.259061098098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8859 , TFLOPS: 100.78412948039265, Tokens per sec: 82353.30661272722, Loss: 2.2645726203918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8860 , TFLOPS: 102.01285565707305, Tokens per sec: 83357.33040191847, Loss: 2.2680931091308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8861 , TFLOPS: 102.20851795697814, Tokens per sec: 83517.2111039666, Loss: 2.2483444213867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8862 , TFLOPS: 101.5095702296124, Tokens per sec: 82946.0829234209, Loss: 2.2530317306518555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8863 , TFLOPS: 101.66967041994907, Tokens per sec: 83076.90491029054, Loss: 2.2562873363494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8864 , TFLOPS: 102.74065771239344, Tokens per sec: 83952.0361966124, Loss: 2.2575042247772217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8865 , TFLOPS: 100.99589804573874, Tokens per sec: 82526.34815887935, Loss: 2.2700743675231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8866 , TFLOPS: 100.66088787493226, Tokens per sec: 82252.60272438437, Loss: 2.2732582092285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8867 , TFLOPS: 102.68763001937184, Tokens per sec: 83908.70590359019, Loss: 2.2499310970306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8868 , TFLOPS: 100.45207947231849, Tokens per sec: 82081.98000340219, Loss: 2.262497663497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8869 , TFLOPS: 100.91338344517929, Tokens per sec: 82458.92335464746, Loss: 2.271740198135376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8870 , TFLOPS: 101.85164550231225, Tokens per sec: 83225.60143454485, Loss: 2.246103048324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8871 , TFLOPS: 101.55203327935442, Tokens per sec: 82980.78057446124, Loss: 2.253089427947998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8872 , TFLOPS: 100.866184452902, Tokens per sec: 82420.35584304752, Loss: 2.2837753295898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8873 , TFLOPS: 102.12983327221004, Tokens per sec: 83452.9158225186, Loss: 2.2551381587982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8874 , TFLOPS: 100.95334829334966, Tokens per sec: 82491.57966087417, Loss: 2.266094207763672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8875 , TFLOPS: 101.70785908825759, Tokens per sec: 83108.1098542292, Loss: 2.261667013168335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8876 , TFLOPS: 102.28581426040255, Tokens per sec: 83580.37190327856, Loss: 2.2701621055603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8877 , TFLOPS: 99.5993376113827, Tokens per sec: 81385.1826773032, Loss: 2.2818233966827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8878 , TFLOPS: 102.34350464023728, Tokens per sec: 83627.51219772402, Loss: 2.2606234550476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8879 , TFLOPS: 100.96434839710753, Tokens per sec: 82500.56812882259, Loss: 2.2642621994018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8880 , TFLOPS: 100.87447509083269, Tokens per sec: 82427.13033672069, Loss: 2.2643587589263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8881 , TFLOPS: 102.27746515609462, Tokens per sec: 83573.54963522348, Loss: 2.2832915782928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8882 , TFLOPS: 100.90748817512491, Tokens per sec: 82454.10617772826, Loss: 2.2547919750213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8883 , TFLOPS: 102.74379505261842, Tokens per sec: 83954.59979807257, Loss: 2.272749900817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8884 , TFLOPS: 101.98245034929596, Tokens per sec: 83332.48544223155, Loss: 2.2766337394714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8885 , TFLOPS: 102.86811676401278, Tokens per sec: 84056.18626877846, Loss: 2.253511667251587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8886 , TFLOPS: 102.70538953797302, Tokens per sec: 83923.2176634099, Loss: 2.278838634490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8887 , TFLOPS: 102.03302137677056, Tokens per sec: 83373.80833059513, Loss: 2.269397020339966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8888 , TFLOPS: 102.78737600286927, Tokens per sec: 83990.210914396, Loss: 2.226170301437378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8889 , TFLOPS: 103.36471688717516, Tokens per sec: 84461.97101303887, Loss: 2.2630152702331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8890 , TFLOPS: 101.44808557972502, Tokens per sec: 82895.84223324222, Loss: 2.2605714797973633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8891 , TFLOPS: 103.34069278528959, Tokens per sec: 84442.34030095275, Loss: 2.2566394805908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8892 , TFLOPS: 102.78786487596707, Tokens per sec: 83990.61038518892, Loss: 2.2520596981048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8893 , TFLOPS: 99.90032238421026, Tokens per sec: 81631.12508321792, Loss: 2.2652621269226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8894 , TFLOPS: 102.1498106131536, Tokens per sec: 83469.23982206611, Loss: 2.267679214477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8895 , TFLOPS: 102.83491284845606, Tokens per sec: 84029.05449463222, Loss: 2.2852468490600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8896 , TFLOPS: 102.61609505625403, Tokens per sec: 83850.25284375285, Loss: 2.2593157291412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8897 , TFLOPS: 101.58966682299658, Tokens per sec: 83011.53191174485, Loss: 2.2748513221740723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8898 , TFLOPS: 100.9116509510715, Tokens per sec: 82457.5076891136, Loss: 2.2482807636260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8899 , TFLOPS: 102.05774463541084, Tokens per sec: 83394.0103416634, Loss: 2.2616329193115234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8900 , TFLOPS: 102.31565680664937, Tokens per sec: 83604.75701603232, Loss: 2.298239231109619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8901 , TFLOPS: 101.08999125138683, Tokens per sec: 82603.23413938901, Loss: 2.2733678817749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8902 , TFLOPS: 102.7996021777773, Tokens per sec: 84000.20123664307, Loss: 2.260053873062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8903 , TFLOPS: 101.12381690403845, Tokens per sec: 82630.87395092042, Loss: 2.2836127281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8904 , TFLOPS: 102.11233727624526, Tokens per sec: 83438.61939382879, Loss: 2.264052152633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8905 , TFLOPS: 102.13780160393591, Tokens per sec: 83459.42695149491, Loss: 2.2447681427001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8906 , TFLOPS: 100.79042093635492, Tokens per sec: 82358.44752335046, Loss: 2.2693803310394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8907 , TFLOPS: 102.13836080394618, Tokens per sec: 83459.8838882184, Loss: 2.2792818546295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8908 , TFLOPS: 101.10280876983649, Tokens per sec: 82613.7076636665, Loss: 2.272719144821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8909 , TFLOPS: 101.50373970345295, Tokens per sec: 82941.31865040492, Loss: 2.2732014656066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8910 , TFLOPS: 100.97721763078285, Tokens per sec: 82511.08391094224, Loss: 2.276899576187134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8911 , TFLOPS: 101.42647594251315, Tokens per sec: 82878.18444239486, Loss: 2.2704577445983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8912 , TFLOPS: 101.1200667122378, Tokens per sec: 82627.80957266157, Loss: 2.266667366027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8913 , TFLOPS: 102.25260443061788, Tokens per sec: 83553.23529646442, Loss: 2.247596502304077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8914 , TFLOPS: 100.90765672359362, Tokens per sec: 82454.24390302088, Loss: 2.2808802127838135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8915 , TFLOPS: 101.20356771296457, Tokens per sec: 82696.0403898624, Loss: 2.2729084491729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8916 , TFLOPS: 101.66574078050499, Tokens per sec: 83073.69390074286, Loss: 2.2483131885528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8917 , TFLOPS: 102.13903014050832, Tokens per sec: 83460.43082035359, Loss: 2.2629382610321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8918 , TFLOPS: 101.52828999375942, Tokens per sec: 82961.37932459502, Loss: 2.250209331512451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8919 , TFLOPS: 102.25202894984987, Tokens per sec: 83552.76505631469, Loss: 2.25706148147583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8920 , TFLOPS: 101.62393273408586, Tokens per sec: 83039.53146977893, Loss: 2.243170738220215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8921 , TFLOPS: 102.49188384783284, Tokens per sec: 83748.75666787094, Loss: 2.2681961059570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8922 , TFLOPS: 100.95474059138841, Tokens per sec: 82492.71734344258, Loss: 2.279939651489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8923 , TFLOPS: 102.8787261482522, Tokens per sec: 84064.85546975031, Loss: 2.25726318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8924 , TFLOPS: 102.19474564805759, Tokens per sec: 83505.95739581685, Loss: 2.270665168762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8925 , TFLOPS: 101.37551548630158, Tokens per sec: 82836.54334178554, Loss: 2.2473509311676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8926 , TFLOPS: 102.29253799846968, Tokens per sec: 83585.86604273763, Loss: 2.252873659133911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8927 , TFLOPS: 103.29294703795885, Tokens per sec: 84403.32602172393, Loss: 2.253736734390259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8928 , TFLOPS: 101.12096190596901, Tokens per sec: 82628.54105850367, Loss: 2.2702012062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8929 , TFLOPS: 103.54773769370017, Tokens per sec: 84611.52202542515, Loss: 2.2768611907958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8930 , TFLOPS: 102.79477384728472, Tokens per sec: 83996.2558835053, Loss: 2.281249761581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8931 , TFLOPS: 100.83272197558786, Tokens per sec: 82393.01279143334, Loss: 2.2749717235565186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8932 , TFLOPS: 102.10154438921423, Tokens per sec: 83429.8002480021, Loss: 2.257352352142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8933 , TFLOPS: 102.86571293175469, Tokens per sec: 84054.2220355602, Loss: 2.2625815868377686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8934 , TFLOPS: 101.94758748469862, Tokens per sec: 83303.99809811909, Loss: 2.2667622566223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8935 , TFLOPS: 101.57099814452151, Tokens per sec: 82996.27725398833, Loss: 2.2512259483337402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8936 , TFLOPS: 100.93210453229467, Tokens per sec: 82474.22083685307, Loss: 2.2650585174560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8937 , TFLOPS: 102.78919948185724, Tokens per sec: 83991.70092601758, Loss: 2.238283634185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8938 , TFLOPS: 102.28711166586734, Tokens per sec: 83581.43204668202, Loss: 2.2885189056396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8939 , TFLOPS: 100.94965594983941, Tokens per sec: 82488.56255194257, Loss: 2.2626404762268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8940 , TFLOPS: 102.75304065349734, Tokens per sec: 83962.15461655366, Loss: 2.2667105197906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8941 , TFLOPS: 99.89012644816285, Tokens per sec: 81622.79372140685, Loss: 2.263091564178467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8942 , TFLOPS: 102.7381100352134, Tokens per sec: 83949.95442400555, Loss: 2.2731032371520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8943 , TFLOPS: 102.22782906243651, Tokens per sec: 83532.99070534873, Loss: 2.2531442642211914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8944 , TFLOPS: 100.90546024701165, Tokens per sec: 82452.4491054634, Loss: 2.2470197677612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8945 , TFLOPS: 101.49821470783267, Tokens per sec: 82936.80403425754, Loss: 2.2713208198547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8946 , TFLOPS: 101.75238165586325, Tokens per sec: 83144.49039033263, Loss: 2.2703535556793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8947 , TFLOPS: 101.47300989352556, Tokens per sec: 82916.20853165757, Loss: 2.257551670074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8948 , TFLOPS: 100.94375212564448, Tokens per sec: 82483.73838521494, Loss: 2.2518277168273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8949 , TFLOPS: 101.59323670085021, Tokens per sec: 83014.44895083588, Loss: 2.255079507827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8950 , TFLOPS: 101.05603958536196, Tokens per sec: 82575.49135908645, Loss: 2.2777395248413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8951 , TFLOPS: 102.20323739818727, Tokens per sec: 83512.89622344491, Loss: 2.262312173843384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8952 , TFLOPS: 100.38545101057355, Tokens per sec: 82027.5361721412, Loss: 2.2484843730926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8953 , TFLOPS: 101.19665309885634, Tokens per sec: 82690.39028067657, Loss: 2.2584025859832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8954 , TFLOPS: 102.16880290906076, Tokens per sec: 83484.75891595703, Loss: 2.2621335983276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8955 , TFLOPS: 101.31878699236752, Tokens per sec: 82790.18902907067, Loss: 2.2567219734191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8956 , TFLOPS: 102.11322116758359, Tokens per sec: 83439.34164419472, Loss: 2.273578405380249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8957 , TFLOPS: 101.58267153535546, Tokens per sec: 83005.81588213857, Loss: 2.2701632976531982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8958 , TFLOPS: 102.25233082314442, Tokens per sec: 83553.01172475459, Loss: 2.251115083694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8959 , TFLOPS: 102.80790534386125, Tokens per sec: 84006.98596739279, Loss: 2.2685580253601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8960 , TFLOPS: 100.90822692510066, Tokens per sec: 82454.70982935056, Loss: 2.2581095695495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8961 , TFLOPS: 102.84909434249175, Tokens per sec: 84040.64255847293, Loss: 2.2702345848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8962 , TFLOPS: 101.99336253045935, Tokens per sec: 83341.4020663647, Loss: 2.2649264335632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8963 , TFLOPS: 99.62014591237929, Tokens per sec: 81402.18567570082, Loss: 2.292018413543701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8964 , TFLOPS: 102.81821018804175, Tokens per sec: 84015.40632084296, Loss: 2.255446434020996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8965 , TFLOPS: 102.74931541008719, Tokens per sec: 83959.11062426693, Loss: 2.2794406414031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8966 , TFLOPS: 101.3617330052225, Tokens per sec: 82825.2813217032, Loss: 2.259249687194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8967 , TFLOPS: 102.69554409688594, Tokens per sec: 83915.17270005344, Loss: 2.2468903064727783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8968 , TFLOPS: 102.59106904917856, Tokens per sec: 83829.80344914476, Loss: 2.2601191997528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8969 , TFLOPS: 100.8616794293196, Tokens per sec: 82416.67467230871, Loss: 2.2651331424713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8970 , TFLOPS: 102.07726529564636, Tokens per sec: 83409.96117564832, Loss: 2.2893948554992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8971 , TFLOPS: 102.89538447652544, Tokens per sec: 84078.4674186059, Loss: 2.2653627395629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8972 , TFLOPS: 102.03339225740001, Tokens per sec: 83374.11138670475, Loss: 2.268272876739502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8973 , TFLOPS: 101.38263901190733, Tokens per sec: 82842.36415793386, Loss: 2.26654052734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8974 , TFLOPS: 101.57154609853032, Tokens per sec: 82996.72500131452, Loss: 2.2608447074890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8975 , TFLOPS: 102.28449544398634, Tokens per sec: 83579.29426443552, Loss: 2.2524404525756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8976 , TFLOPS: 101.59935630854756, Tokens per sec: 83019.44943981811, Loss: 2.270718574523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8977 , TFLOPS: 101.12203921986058, Tokens per sec: 82629.42135941701, Loss: 2.2678334712982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8978 , TFLOPS: 102.75071799798671, Tokens per sec: 83960.25671494546, Loss: 2.2563891410827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8979 , TFLOPS: 100.96292553211248, Tokens per sec: 82499.40546920731, Loss: 2.2520363330841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8980 , TFLOPS: 102.69706883082651, Tokens per sec: 83916.41859940643, Loss: 2.2627451419830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8981 , TFLOPS: 101.59712223630908, Tokens per sec: 83017.62392188191, Loss: 2.24656343460083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8982 , TFLOPS: 100.74399523513425, Tokens per sec: 82320.5118877792, Loss: 2.241345167160034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8983 , TFLOPS: 101.57443922994716, Tokens per sec: 82999.08905347105, Loss: 2.280775785446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8984 , TFLOPS: 101.80668981330382, Tokens per sec: 83188.86698379368, Loss: 2.2599868774414062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8985 , TFLOPS: 101.44635240628558, Tokens per sec: 82894.42601260901, Loss: 2.250985860824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8986 , TFLOPS: 100.74956882257761, Tokens per sec: 82325.06620955608, Loss: 2.2665083408355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8987 , TFLOPS: 101.40942154147129, Tokens per sec: 82864.2488522842, Loss: 2.2727975845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8988 , TFLOPS: 100.51646594888334, Tokens per sec: 82134.59185085869, Loss: 2.271914005279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8989 , TFLOPS: 102.79402107616306, Tokens per sec: 83995.64077484375, Loss: 2.2385387420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8990 , TFLOPS: 101.22763922717886, Tokens per sec: 82715.70984378281, Loss: 2.27449369430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8991 , TFLOPS: 100.47098450649645, Tokens per sec: 82097.42779348788, Loss: 2.2469377517700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8992 , TFLOPS: 102.22244590086342, Tokens per sec: 83528.59198545244, Loss: 2.278850793838501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8993 , TFLOPS: 102.74430446484246, Tokens per sec: 83955.01605191425, Loss: 2.25925612449646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8994 , TFLOPS: 101.33932360460226, Tokens per sec: 82806.9700235871, Loss: 2.2741289138793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8995 , TFLOPS: 102.80710555119197, Tokens per sec: 84006.33243621422, Loss: 2.245426893234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8996 , TFLOPS: 102.7167711815096, Tokens per sec: 83932.51789733321, Loss: 2.2428677082061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8997 , TFLOPS: 101.98112841698267, Tokens per sec: 83331.4052573088, Loss: 2.2631101608276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8998 , TFLOPS: 102.14708155325539, Tokens per sec: 83467.00983697102, Loss: 2.2561261653900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8999 , TFLOPS: 101.44284660302209, Tokens per sec: 82891.56133051502, Loss: 2.2737927436828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9000 , TFLOPS: 102.55224099601674, Tokens per sec: 83798.07604738323, Loss: 2.277829647064209 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/optimizer.pt +[rank0]:[2024-08-28 13:56:59,208] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007679091999307275, 'preprocessing_with_comm': 0.001575362985022366, 'state_converting': 3.3705916710023303, : 3.38145327399252}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000, took 15.85s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007000 + eval ppl=7.611457824707031, eval loss=2.0296547412872314 +------------------------------------------------------------------ +iteration: 9001 , TFLOPS: 100.60330743617075, Tokens per sec: 82205.55226561999, Loss: 2.2516274452209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9002 , TFLOPS: 100.424760823655, Tokens per sec: 82059.65723233468, Loss: 2.2491347789764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9003 , TFLOPS: 101.07881652248594, Tokens per sec: 82594.10298074096, Loss: 2.2731575965881348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9004 , TFLOPS: 101.18669257318506, Tokens per sec: 82682.25127874382, Loss: 2.260730028152466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9005 , TFLOPS: 100.68544400381367, Tokens per sec: 82272.66816942437, Loss: 2.278425931930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9006 , TFLOPS: 103.45862388810134, Tokens per sec: 84538.7048408769, Loss: 2.283294916152954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9007 , TFLOPS: 102.87215445104907, Tokens per sec: 84059.4855667951, Loss: 2.2594757080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9008 , TFLOPS: 102.95926575762763, Tokens per sec: 84130.66645784497, Loss: 2.263331174850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9009 , TFLOPS: 102.67975137221445, Tokens per sec: 83902.26805818347, Loss: 2.2587554454803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9010 , TFLOPS: 102.76413919715097, Tokens per sec: 83971.22352227497, Loss: 2.267487049102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9011 , TFLOPS: 103.39318153547534, Tokens per sec: 84485.23020991017, Loss: 2.262787342071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9012 , TFLOPS: 103.44123255139738, Tokens per sec: 84524.49393196327, Loss: 2.262716054916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9013 , TFLOPS: 101.18833842927943, Tokens per sec: 82683.59615012663, Loss: 2.2549350261688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9014 , TFLOPS: 101.23470439642294, Tokens per sec: 82721.48297544576, Loss: 2.2757558822631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9015 , TFLOPS: 102.77280556284465, Tokens per sec: 83978.30503277539, Loss: 2.2861838340759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9016 , TFLOPS: 103.32190073935034, Tokens per sec: 84426.9848364655, Loss: 2.2623403072357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9017 , TFLOPS: 102.68841409683341, Tokens per sec: 83909.346593468, Loss: 2.263045072555542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9018 , TFLOPS: 101.79211615789953, Tokens per sec: 83176.95847480356, Loss: 2.2706761360168457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9019 , TFLOPS: 100.81540872146832, Tokens per sec: 82378.86568580945, Loss: 2.2614612579345703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9020 , TFLOPS: 102.62059783298363, Tokens per sec: 83853.9321785304, Loss: 2.266968011856079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9021 , TFLOPS: 101.83321856511459, Tokens per sec: 83210.54430980909, Loss: 2.270254611968994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9022 , TFLOPS: 103.39331536676009, Tokens per sec: 84485.33956689807, Loss: 2.268747329711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9023 , TFLOPS: 100.88025588298488, Tokens per sec: 82431.85397079907, Loss: 2.2607569694519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9024 , TFLOPS: 101.05969019035405, Tokens per sec: 82578.4743624004, Loss: 2.2756545543670654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9025 , TFLOPS: 103.4856617160688, Tokens per sec: 84560.79814612473, Loss: 2.2541792392730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9026 , TFLOPS: 101.47261494865165, Tokens per sec: 82915.8858120343, Loss: 2.276902198791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9027 , TFLOPS: 100.30922329441381, Tokens per sec: 81965.2485429911, Loss: 2.25300931930542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9028 , TFLOPS: 102.57041253313733, Tokens per sec: 83812.92447814145, Loss: 2.2520978450775146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9029 , TFLOPS: 101.53286492147767, Tokens per sec: 82965.11761580272, Loss: 2.239914655685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9030 , TFLOPS: 99.91584896173974, Tokens per sec: 81643.81224940694, Loss: 2.252805471420288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9031 , TFLOPS: 102.66079869664799, Tokens per sec: 83886.78133909285, Loss: 2.27140736579895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9032 , TFLOPS: 100.30723836614344, Tokens per sec: 81963.62660700479, Loss: 2.252959966659546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9033 , TFLOPS: 101.4017313699756, Tokens per sec: 82857.96501519246, Loss: 2.2689967155456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9034 , TFLOPS: 100.50415592142599, Tokens per sec: 82124.533010535, Loss: 2.2642691135406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9035 , TFLOPS: 100.90928043154175, Tokens per sec: 82455.57067658323, Loss: 2.2564120292663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9036 , TFLOPS: 100.39427443164618, Tokens per sec: 82034.74601663473, Loss: 2.255760669708252 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9037 , TFLOPS: 102.19644470994722, Tokens per sec: 83507.3457430246, Loss: 2.2625794410705566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9038 , TFLOPS: 100.41635143037402, Tokens per sec: 82052.78570060775, Loss: 2.262593984603882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9039 , TFLOPS: 100.05916825866862, Tokens per sec: 81760.92213629444, Loss: 2.259885311126709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9040 , TFLOPS: 101.04781582098417, Tokens per sec: 82568.77150951483, Loss: 2.2466766834259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9041 , TFLOPS: 101.46588004924303, Tokens per sec: 82910.38255235579, Loss: 2.264280080795288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9042 , TFLOPS: 101.67365329899039, Tokens per sec: 83080.15942328362, Loss: 2.269301414489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9043 , TFLOPS: 101.20042724807362, Tokens per sec: 82693.47423515725, Loss: 2.2628393173217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9044 , TFLOPS: 103.3054189767641, Tokens per sec: 84413.51716397797, Loss: 2.2589807510375977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9045 , TFLOPS: 102.53798118326897, Tokens per sec: 83786.42398730679, Loss: 2.2642760276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9046 , TFLOPS: 103.42730392946642, Tokens per sec: 84513.11249642886, Loss: 2.247199535369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9047 , TFLOPS: 103.45031033784355, Tokens per sec: 84531.91162495145, Loss: 2.248649835586548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9048 , TFLOPS: 102.75184961068143, Tokens per sec: 83961.1813848086, Loss: 2.2575502395629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9049 , TFLOPS: 102.72017794748187, Tokens per sec: 83935.30165350711, Loss: 2.271007776260376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9050 , TFLOPS: 103.32991228339671, Tokens per sec: 84433.53127534129, Loss: 2.266378402709961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9051 , TFLOPS: 100.69416206090486, Tokens per sec: 82279.79191829634, Loss: 2.262967348098755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9052 , TFLOPS: 102.05259751473919, Tokens per sec: 83389.8044968639, Loss: 2.2585270404815674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9053 , TFLOPS: 102.18112154099029, Tokens per sec: 83494.82478721645, Loss: 2.2685928344726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9054 , TFLOPS: 103.47810413875041, Tokens per sec: 84554.62265514852, Loss: 2.252121686935425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9055 , TFLOPS: 102.48578272702042, Tokens per sec: 83743.7712849983, Loss: 2.2643113136291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9056 , TFLOPS: 102.64875062125675, Tokens per sec: 83876.9365465456, Loss: 2.2749791145324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9057 , TFLOPS: 100.17194502203203, Tokens per sec: 81853.07493277085, Loss: 2.258542537689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9058 , TFLOPS: 102.76116843364484, Tokens per sec: 83968.79603493985, Loss: 2.266508102416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9059 , TFLOPS: 102.03048454133751, Tokens per sec: 83371.73541705881, Loss: 2.2594680786132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9060 , TFLOPS: 103.45202918996658, Tokens per sec: 84533.31614326836, Loss: 2.245518684387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9061 , TFLOPS: 101.07797516864788, Tokens per sec: 82593.41548886152, Loss: 2.2847795486450195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9062 , TFLOPS: 100.99802394042266, Tokens per sec: 82528.08528215099, Loss: 2.2567474842071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9063 , TFLOPS: 103.38392718329675, Tokens per sec: 84477.66824051682, Loss: 2.270537853240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9064 , TFLOPS: 101.51116410987633, Tokens per sec: 82947.38532401469, Loss: 2.250235080718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9065 , TFLOPS: 100.27101555586982, Tokens per sec: 81934.02801627212, Loss: 2.268327236175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9066 , TFLOPS: 102.80352411733425, Tokens per sec: 84003.40595441477, Loss: 2.2510530948638916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9067 , TFLOPS: 101.46285324126379, Tokens per sec: 82907.90926963899, Loss: 2.2672653198242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9068 , TFLOPS: 100.3360790253896, Tokens per sec: 81987.19305209944, Loss: 2.2623441219329834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9069 , TFLOPS: 102.81648403558475, Tokens per sec: 84013.99583723524, Loss: 2.256575345993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9070 , TFLOPS: 100.15976001716129, Tokens per sec: 81843.11825162102, Loss: 2.251887798309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9071 , TFLOPS: 100.63083459755491, Tokens per sec: 82228.04541779916, Loss: 2.282942533493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9072 , TFLOPS: 101.10524416773721, Tokens per sec: 82615.69768998389, Loss: 2.2398242950439453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9073 , TFLOPS: 101.08908479984572, Tokens per sec: 82602.49345450063, Loss: 2.2686047554016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9074 , TFLOPS: 100.30649477921243, Tokens per sec: 81963.01900298183, Loss: 2.264373779296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9075 , TFLOPS: 101.69699853790331, Tokens per sec: 83099.23541895927, Loss: 2.259706497192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9076 , TFLOPS: 100.87031795419283, Tokens per sec: 82423.73343335804, Loss: 2.2598023414611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9077 , TFLOPS: 99.89812502707419, Tokens per sec: 81629.32956613705, Loss: 2.2444162368774414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9078 , TFLOPS: 100.81550954076349, Tokens per sec: 82378.94806785084, Loss: 2.2701964378356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9079 , TFLOPS: 101.90717650928008, Tokens per sec: 83270.97724983355, Loss: 2.245378255844116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9080 , TFLOPS: 101.87928384436168, Tokens per sec: 83248.18543530766, Loss: 2.2741761207580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9081 , TFLOPS: 100.66359772491852, Tokens per sec: 82254.8170125659, Loss: 2.2649943828582764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9082 , TFLOPS: 103.4196324042375, Tokens per sec: 84506.84389568168, Loss: 2.245753049850464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9083 , TFLOPS: 103.4131155616125, Tokens per sec: 84501.51881581433, Loss: 2.266655445098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9084 , TFLOPS: 103.4270281100075, Tokens per sec: 84512.88711724874, Loss: 2.2668192386627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9085 , TFLOPS: 103.36904692168682, Tokens per sec: 84465.50919570334, Loss: 2.2678890228271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9086 , TFLOPS: 101.67728860186946, Tokens per sec: 83083.12992284713, Loss: 2.2728006839752197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9087 , TFLOPS: 102.76435978234728, Tokens per sec: 83971.40376811725, Loss: 2.275203227996826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9088 , TFLOPS: 103.37994738258824, Tokens per sec: 84474.41624290872, Loss: 2.277998685836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9089 , TFLOPS: 101.33250488247305, Tokens per sec: 82801.3982702058, Loss: 2.267543315887451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9090 , TFLOPS: 102.00166605233787, Tokens per sec: 83348.1870878433, Loss: 2.258854389190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9091 , TFLOPS: 102.20899554891001, Tokens per sec: 83517.60135662845, Loss: 2.2576303482055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9092 , TFLOPS: 102.41595802621238, Tokens per sec: 83686.71572452034, Loss: 2.2662582397460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9093 , TFLOPS: 103.25229630766634, Tokens per sec: 84370.10926355905, Loss: 2.2638375759124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9094 , TFLOPS: 102.61585574082237, Tokens per sec: 83850.05729295312, Loss: 2.2568089962005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9095 , TFLOPS: 100.26453847435093, Tokens per sec: 81928.73542622813, Loss: 2.2735726833343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9096 , TFLOPS: 102.82285734338828, Tokens per sec: 84019.20363110503, Loss: 2.2685277462005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9097 , TFLOPS: 102.07373857978104, Tokens per sec: 83407.07940532902, Loss: 2.2392146587371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9098 , TFLOPS: 103.36494545488787, Tokens per sec: 84462.15778160078, Loss: 2.2697157859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9099 , TFLOPS: 101.04286380777026, Tokens per sec: 82564.72509204158, Loss: 2.2496159076690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9100 , TFLOPS: 101.8168585995512, Tokens per sec: 83197.17616080426, Loss: 2.258676767349243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9101 , TFLOPS: 102.6928539298557, Tokens per sec: 83912.97449532198, Loss: 2.2471396923065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9102 , TFLOPS: 101.55689325778474, Tokens per sec: 82984.75178794363, Loss: 2.256251335144043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9103 , TFLOPS: 101.16686493448479, Tokens per sec: 82666.049604753, Loss: 2.297311544418335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9104 , TFLOPS: 102.03318186014366, Tokens per sec: 83373.93946569048, Loss: 2.250239133834839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9105 , TFLOPS: 101.33065158758725, Tokens per sec: 82799.88389524657, Loss: 2.2660512924194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9106 , TFLOPS: 101.06072653244355, Tokens per sec: 82579.32118419958, Loss: 2.2713100910186768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9107 , TFLOPS: 101.91486356441737, Tokens per sec: 83277.25854046864, Loss: 2.2808241844177246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9108 , TFLOPS: 100.81503016546645, Tokens per sec: 82378.55635795543, Loss: 2.271195411682129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9109 , TFLOPS: 100.0523711037285, Tokens per sec: 81755.36800601854, Loss: 2.2610886096954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9110 , TFLOPS: 101.69008046254261, Tokens per sec: 83093.58248149494, Loss: 2.2663841247558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9111 , TFLOPS: 100.44182970564464, Tokens per sec: 82073.60465520053, Loss: 2.241905689239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9112 , TFLOPS: 101.19832222156754, Tokens per sec: 82691.75416381091, Loss: 2.2726991176605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9113 , TFLOPS: 101.4417021277012, Tokens per sec: 82890.62615026889, Loss: 2.256202220916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9114 , TFLOPS: 100.8524167820454, Tokens per sec: 82409.1059247792, Loss: 2.2751076221466064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9115 , TFLOPS: 100.29440941502408, Tokens per sec: 81953.14374079858, Loss: 2.267160177230835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9116 , TFLOPS: 101.1131570849, Tokens per sec: 82622.16353829422, Loss: 2.2586565017700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9117 , TFLOPS: 101.19929666859849, Tokens per sec: 82692.55040956441, Loss: 2.260545015335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9118 , TFLOPS: 101.25218982084915, Tokens per sec: 82735.7707658593, Loss: 2.295984983444214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9119 , TFLOPS: 100.60709668804603, Tokens per sec: 82208.6485608713, Loss: 2.282904624938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9120 , TFLOPS: 103.45215549484429, Tokens per sec: 84533.41935023532, Loss: 2.2637529373168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9121 , TFLOPS: 103.36311556438444, Tokens per sec: 84460.66253096485, Loss: 2.250354051589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9122 , TFLOPS: 103.35779515808898, Tokens per sec: 84456.31509002138, Loss: 2.2688984870910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9123 , TFLOPS: 102.81010427008565, Tokens per sec: 84008.78276661613, Loss: 2.2539477348327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9124 , TFLOPS: 102.70387656578107, Tokens per sec: 83921.98137488449, Loss: 2.254354953765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9125 , TFLOPS: 103.33966012897164, Tokens per sec: 84441.49649089243, Loss: 2.255931854248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9126 , TFLOPS: 102.74089452044224, Tokens per sec: 83952.22969856505, Loss: 2.2658214569091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9127 , TFLOPS: 100.42513819438199, Tokens per sec: 82059.96559167007, Loss: 2.267500400543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9128 , TFLOPS: 101.87727763138655, Tokens per sec: 83246.54610704115, Loss: 2.2522923946380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9129 , TFLOPS: 102.68154615377739, Tokens per sec: 83903.73462040034, Loss: 2.266585111618042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9130 , TFLOPS: 102.2439782186808, Tokens per sec: 83546.1865966322, Loss: 2.2609684467315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9131 , TFLOPS: 103.2820272391889, Tokens per sec: 84394.40317304838, Loss: 2.264310836791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9132 , TFLOPS: 101.89625620969866, Tokens per sec: 83262.05399193206, Loss: 2.2858545780181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9133 , TFLOPS: 100.66767748868732, Tokens per sec: 82258.15069256384, Loss: 2.2544493675231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9134 , TFLOPS: 101.57202606410117, Tokens per sec: 82997.11719353788, Loss: 2.2647759914398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9135 , TFLOPS: 102.51123560882128, Tokens per sec: 83764.56948993324, Loss: 2.260402202606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9136 , TFLOPS: 102.74505435147033, Tokens per sec: 83955.62880358168, Loss: 2.2570900917053223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9137 , TFLOPS: 101.11066116093978, Tokens per sec: 82620.12405457553, Loss: 2.255582332611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9138 , TFLOPS: 101.59380495074475, Tokens per sec: 83014.91328245276, Loss: 2.269521713256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9139 , TFLOPS: 101.89327973852619, Tokens per sec: 83259.62184071584, Loss: 2.263716459274292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9140 , TFLOPS: 101.92956344764443, Tokens per sec: 83289.27019346211, Loss: 2.2500600814819336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9141 , TFLOPS: 99.88031213729451, Tokens per sec: 81614.7741953535, Loss: 2.2746176719665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9142 , TFLOPS: 102.89807528157054, Tokens per sec: 84080.66614467575, Loss: 2.272019147872925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9143 , TFLOPS: 101.69855107446348, Tokens per sec: 83100.50403654845, Loss: 2.258159875869751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9144 , TFLOPS: 100.16343024429855, Tokens per sec: 81846.11728869494, Loss: 2.2748849391937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9145 , TFLOPS: 102.32582149249339, Tokens per sec: 83613.06284249775, Loss: 2.2584962844848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9146 , TFLOPS: 101.22172395998535, Tokens per sec: 82710.87632668646, Loss: 2.2772789001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9147 , TFLOPS: 100.86985542054703, Tokens per sec: 82423.35548520938, Loss: 2.2626335620880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9148 , TFLOPS: 101.50633871266153, Tokens per sec: 82943.4423677337, Loss: 2.250180959701538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9149 , TFLOPS: 100.81061889877876, Tokens per sec: 82374.9517983888, Loss: 2.2629292011260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9150 , TFLOPS: 100.46118892264747, Tokens per sec: 82089.42356976411, Loss: 2.2656965255737305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9151 , TFLOPS: 100.93036999308971, Tokens per sec: 82472.8035002177, Loss: 2.277829647064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9152 , TFLOPS: 101.54636801111036, Tokens per sec: 82976.15133794205, Loss: 2.273026943206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9153 , TFLOPS: 100.90150312166935, Tokens per sec: 82449.21563648073, Loss: 2.235858917236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9154 , TFLOPS: 101.07226513890767, Tokens per sec: 82588.74967657155, Loss: 2.2877023220062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9155 , TFLOPS: 101.87409287633477, Tokens per sec: 83243.94376170546, Loss: 2.2544708251953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9156 , TFLOPS: 102.23083391828256, Tokens per sec: 83535.44605041239, Loss: 2.238738536834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9157 , TFLOPS: 101.24793867094647, Tokens per sec: 82732.29704183947, Loss: 2.2602639198303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9158 , TFLOPS: 102.72583594008492, Tokens per sec: 83939.92494491274, Loss: 2.275416135787964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9159 , TFLOPS: 102.78730594408418, Tokens per sec: 83990.15366755922, Loss: 2.262523651123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9160 , TFLOPS: 103.42841470337949, Tokens per sec: 84514.02013838709, Loss: 2.2656314373016357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9161 , TFLOPS: 103.28189283890364, Tokens per sec: 84394.29335111553, Loss: 2.2751946449279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9162 , TFLOPS: 101.3690815301097, Tokens per sec: 82831.28599055638, Loss: 2.2559103965759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9163 , TFLOPS: 103.35218662663769, Tokens per sec: 84451.73221459611, Loss: 2.2735178470611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9164 , TFLOPS: 102.02984357828595, Tokens per sec: 83371.21166964952, Loss: 2.2858541011810303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9165 , TFLOPS: 102.02769083613144, Tokens per sec: 83369.45260861857, Loss: 2.272939682006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9166 , TFLOPS: 101.48248122428627, Tokens per sec: 82923.94779983588, Loss: 2.264099359512329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9167 , TFLOPS: 102.70855366551383, Tokens per sec: 83925.80315347538, Loss: 2.2706172466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9168 , TFLOPS: 101.34801059805692, Tokens per sec: 82814.068389562, Loss: 2.2774159908294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9169 , TFLOPS: 103.48933710204108, Tokens per sec: 84563.80139861553, Loss: 2.260084867477417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9170 , TFLOPS: 102.79168410369836, Tokens per sec: 83993.73117448382, Loss: 2.2667596340179443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9171 , TFLOPS: 100.5778180053142, Tokens per sec: 82184.72419551079, Loss: 2.252197027206421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9172 , TFLOPS: 101.5329890105496, Tokens per sec: 82965.21901217772, Loss: 2.2770848274230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9173 , TFLOPS: 101.85542440627763, Tokens per sec: 83228.68927425261, Loss: 2.2759406566619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9174 , TFLOPS: 103.30514683784341, Tokens per sec: 84413.29479226039, Loss: 2.2564096450805664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9175 , TFLOPS: 100.53416561706403, Tokens per sec: 82149.05470535913, Loss: 2.2678534984588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9176 , TFLOPS: 102.23466504591728, Tokens per sec: 83538.57656342472, Loss: 2.2462642192840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9177 , TFLOPS: 101.67149690990604, Tokens per sec: 83078.39738225241, Loss: 2.2546193599700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9178 , TFLOPS: 102.5321695569339, Tokens per sec: 83781.67515782383, Loss: 2.280637741088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9179 , TFLOPS: 101.09294365437161, Tokens per sec: 82605.64662386948, Loss: 2.2718935012817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9180 , TFLOPS: 102.06870713697106, Tokens per sec: 83402.96808388813, Loss: 2.248089551925659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9181 , TFLOPS: 102.08210612913808, Tokens per sec: 83413.91674530854, Loss: 2.2784924507141113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9182 , TFLOPS: 100.78102529713095, Tokens per sec: 82350.7701046753, Loss: 2.2591588497161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9183 , TFLOPS: 100.92670220074776, Tokens per sec: 82469.8064526777, Loss: 2.2734134197235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9184 , TFLOPS: 102.16009064457184, Tokens per sec: 83477.63990036948, Loss: 2.2531561851501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9185 , TFLOPS: 100.2956487682387, Tokens per sec: 81954.15644821404, Loss: 2.2846171855926514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9186 , TFLOPS: 102.17092760773495, Tokens per sec: 83486.49506193832, Loss: 2.279595136642456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9187 , TFLOPS: 99.55145352745372, Tokens per sec: 81346.05535967884, Loss: 2.260751485824585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9188 , TFLOPS: 101.11033925514288, Tokens per sec: 82619.86101706229, Loss: 2.2737812995910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9189 , TFLOPS: 101.51883905262595, Tokens per sec: 82953.656717306, Loss: 2.2591488361358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9190 , TFLOPS: 101.63219469860552, Tokens per sec: 83046.28253366983, Loss: 2.2802579402923584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9191 , TFLOPS: 100.5335473387945, Tokens per sec: 82148.5494942692, Loss: 2.253657817840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9192 , TFLOPS: 101.07308656590637, Tokens per sec: 82589.4208857176, Loss: 2.2638769149780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9193 , TFLOPS: 101.82481671760829, Tokens per sec: 83203.67894392888, Loss: 2.2581794261932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9194 , TFLOPS: 101.68548613380918, Tokens per sec: 83089.82833721832, Loss: 2.2686078548431396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9195 , TFLOPS: 100.44822252684529, Tokens per sec: 82078.82839396944, Loss: 2.267315626144409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9196 , TFLOPS: 101.98480133284234, Tokens per sec: 83334.406491407, Loss: 2.265751600265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9197 , TFLOPS: 103.40553942073649, Tokens per sec: 84495.32815607732, Loss: 2.27296781539917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9198 , TFLOPS: 103.37439057627206, Tokens per sec: 84469.87563342285, Loss: 2.249148368835449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9199 , TFLOPS: 102.77872316659021, Tokens per sec: 83983.14045912855, Loss: 2.2671988010406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9200 , TFLOPS: 102.09369689437099, Tokens per sec: 83423.3878579532, Loss: 2.272850513458252 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/optimizer.pt +[rank0]:[2024-08-28 14:49:54,109] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007709871017141268, 'preprocessing_with_comm': 0.0017602989973966032, 'state_converting': 3.3386707519821357, : 3.349741362995701}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200, took 15.75s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007200 + eval ppl=7.757031440734863, eval loss=2.0485997200012207 +------------------------------------------------------------------ +iteration: 9201 , TFLOPS: 99.27772077999802, Tokens per sec: 81122.381285225, Loss: 2.262188673019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9202 , TFLOPS: 102.28783640707982, Tokens per sec: 83582.02425138325, Loss: 2.260993480682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9203 , TFLOPS: 101.08961696305725, Tokens per sec: 82602.92829875987, Loss: 2.236600399017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9204 , TFLOPS: 100.6078217916944, Tokens per sec: 82209.24106172821, Loss: 2.2551441192626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9205 , TFLOPS: 102.28454658742022, Tokens per sec: 83579.33605505188, Loss: 2.269359827041626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9206 , TFLOPS: 101.05367943053038, Tokens per sec: 82573.56281581779, Loss: 2.2652769088745117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9207 , TFLOPS: 102.20416710905393, Tokens per sec: 83513.65591412694, Loss: 2.236213445663452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9208 , TFLOPS: 101.99444534791587, Tokens per sec: 83342.28686438262, Loss: 2.247453212738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9209 , TFLOPS: 103.42146349684997, Tokens per sec: 84508.34012859184, Loss: 2.2666709423065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9210 , TFLOPS: 100.07769045635325, Tokens per sec: 81776.05710082634, Loss: 2.2813849449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9211 , TFLOPS: 102.5430014276507, Tokens per sec: 83790.52615822369, Loss: 2.270477533340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9212 , TFLOPS: 102.14001302213396, Tokens per sec: 83461.23395823152, Loss: 2.2763779163360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9213 , TFLOPS: 102.47110135035926, Tokens per sec: 83731.7747541962, Loss: 2.269188165664673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9214 , TFLOPS: 101.2883466430277, Tokens per sec: 82765.31543602078, Loss: 2.260038375854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9215 , TFLOPS: 102.73280679604599, Tokens per sec: 83945.62101076433, Loss: 2.2399051189422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9216 , TFLOPS: 102.04787623651883, Tokens per sec: 83385.94661889353, Loss: 2.271371364593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9217 , TFLOPS: 102.78536862782379, Tokens per sec: 83988.5706365719, Loss: 2.2505252361297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9218 , TFLOPS: 101.54720797106033, Tokens per sec: 82976.83769083966, Loss: 2.271573066711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9219 , TFLOPS: 102.20270553937134, Tokens per sec: 83512.46162791514, Loss: 2.272725820541382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9220 , TFLOPS: 102.69311968626349, Tokens per sec: 83913.19165172396, Loss: 2.2607059478759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9221 , TFLOPS: 103.35020302562134, Tokens per sec: 84450.1113631432, Loss: 2.2598559856414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9222 , TFLOPS: 99.72514677146908, Tokens per sec: 81487.98458062568, Loss: 2.2796630859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9223 , TFLOPS: 103.38960743228229, Tokens per sec: 84482.30971818554, Loss: 2.250349521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9224 , TFLOPS: 103.34456815269154, Tokens per sec: 84445.50696341773, Loss: 2.2516961097717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9225 , TFLOPS: 101.04481808137294, Tokens per sec: 82566.32197930981, Loss: 2.264038324356079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9226 , TFLOPS: 101.94740231049954, Tokens per sec: 83303.84678751422, Loss: 2.263197183609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9227 , TFLOPS: 102.10132730905073, Tokens per sec: 83429.62286621236, Loss: 2.2709124088287354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9228 , TFLOPS: 102.01462379759789, Tokens per sec: 83358.7751950573, Loss: 2.264633893966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9229 , TFLOPS: 101.37485204826152, Tokens per sec: 82836.00122948458, Loss: 2.2331056594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9230 , TFLOPS: 101.50748788099442, Tokens per sec: 82944.38138276072, Loss: 2.2524218559265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9231 , TFLOPS: 102.3034876432653, Tokens per sec: 83594.81327936906, Loss: 2.2765564918518066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9232 , TFLOPS: 101.60314952948299, Tokens per sec: 83022.54897829065, Loss: 2.280086040496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9233 , TFLOPS: 100.6255702800171, Tokens per sec: 82223.74380842343, Loss: 2.2522523403167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9234 , TFLOPS: 102.0138099332027, Tokens per sec: 83358.11016550925, Loss: 2.2639236450195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9235 , TFLOPS: 100.88933188529487, Tokens per sec: 82439.27020592366, Loss: 2.271345853805542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9236 , TFLOPS: 101.64329819756183, Tokens per sec: 83055.35548849653, Loss: 2.25700044631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9237 , TFLOPS: 101.27405635230767, Tokens per sec: 82753.63847160866, Loss: 2.268183469772339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9238 , TFLOPS: 100.93040914514945, Tokens per sec: 82472.83549237358, Loss: 2.268582344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9239 , TFLOPS: 101.6981705075059, Tokens per sec: 83100.19306549075, Loss: 2.248464822769165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9240 , TFLOPS: 102.85061836229038, Tokens per sec: 84041.88787428213, Loss: 2.2803080081939697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9241 , TFLOPS: 101.47823496460977, Tokens per sec: 82920.47807174576, Loss: 2.2618579864501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9242 , TFLOPS: 102.9053941375874, Tokens per sec: 84086.64657033123, Loss: 2.2582035064697266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9243 , TFLOPS: 101.90246358046868, Tokens per sec: 83267.12619438018, Loss: 2.2549452781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9244 , TFLOPS: 102.19309451408435, Tokens per sec: 83504.60821174334, Loss: 2.271780490875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9245 , TFLOPS: 102.691872602427, Tokens per sec: 83912.17262741837, Loss: 2.2489638328552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9246 , TFLOPS: 100.83236496461664, Tokens per sec: 82392.72106857848, Loss: 2.2458250522613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9247 , TFLOPS: 102.63242860877682, Tokens per sec: 83863.59940998229, Loss: 2.2664830684661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9248 , TFLOPS: 102.12943242504078, Tokens per sec: 83452.58827997786, Loss: 2.2687041759490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9249 , TFLOPS: 103.33468067360954, Tokens per sec: 84437.42764973346, Loss: 2.2686245441436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9250 , TFLOPS: 102.25290767489138, Tokens per sec: 83553.4830851665, Loss: 2.251675844192505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9251 , TFLOPS: 103.48032806665528, Tokens per sec: 84556.43988388858, Loss: 2.2691729068756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9252 , TFLOPS: 101.39486500266676, Tokens per sec: 82852.3543297084, Loss: 2.2625300884246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9253 , TFLOPS: 102.41053572532303, Tokens per sec: 83682.2850228812, Loss: 2.2810115814208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9254 , TFLOPS: 102.00691554280623, Tokens per sec: 83352.47658163891, Loss: 2.2574329376220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9255 , TFLOPS: 102.7755277219802, Tokens per sec: 83980.52937907964, Loss: 2.262244939804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9256 , TFLOPS: 101.74137796414985, Tokens per sec: 83135.4989905734, Loss: 2.2656445503234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9257 , TFLOPS: 101.50508157217486, Tokens per sec: 82942.41512588036, Loss: 2.2498137950897217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9258 , TFLOPS: 102.8007970198511, Tokens per sec: 84001.17757285955, Loss: 2.2631330490112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9259 , TFLOPS: 103.37854163471285, Tokens per sec: 84473.26757013256, Loss: 2.2633562088012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9260 , TFLOPS: 100.31211736963681, Tokens per sec: 81967.61336635594, Loss: 2.2584710121154785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9261 , TFLOPS: 103.49000372405548, Tokens per sec: 84564.34611262387, Loss: 2.252896547317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9262 , TFLOPS: 103.50106045930303, Tokens per sec: 84573.38085562033, Loss: 2.256638526916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9263 , TFLOPS: 100.05168160949187, Tokens per sec: 81754.80460252862, Loss: 2.231517791748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9264 , TFLOPS: 102.90004628026897, Tokens per sec: 84082.27669845025, Loss: 2.262406826019287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9265 , TFLOPS: 102.14461999607752, Tokens per sec: 83464.99843523485, Loss: 2.2648768424987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9266 , TFLOPS: 102.22516274446068, Tokens per sec: 83530.81198829354, Loss: 2.273922920227051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9267 , TFLOPS: 101.54311304254904, Tokens per sec: 82973.49161934025, Loss: 2.2851874828338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9268 , TFLOPS: 101.41645916558554, Tokens per sec: 82869.99947611253, Loss: 2.3131496906280518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9269 , TFLOPS: 102.15954624106851, Tokens per sec: 83477.19505425273, Loss: 2.2641897201538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9270 , TFLOPS: 101.71376715469314, Tokens per sec: 83112.93748740086, Loss: 2.2631704807281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9271 , TFLOPS: 101.46577248143075, Tokens per sec: 82910.29465592733, Loss: 2.2788033485412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9272 , TFLOPS: 101.31897382213438, Tokens per sec: 82790.34169248259, Loss: 2.2596445083618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9273 , TFLOPS: 101.01618634069618, Tokens per sec: 82542.92624695625, Loss: 2.2347464561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9274 , TFLOPS: 102.37120452804214, Tokens per sec: 83650.14648910791, Loss: 2.259060859680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9275 , TFLOPS: 100.41281795221867, Tokens per sec: 82049.8984046472, Loss: 2.26188588142395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9276 , TFLOPS: 101.66628951254404, Tokens per sec: 83074.14228381758, Loss: 2.2771098613739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9277 , TFLOPS: 101.61249712644143, Tokens per sec: 83030.18714039393, Loss: 2.2549500465393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9278 , TFLOPS: 102.04409516654295, Tokens per sec: 83382.85700928283, Loss: 2.2725071907043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9279 , TFLOPS: 102.14364826077902, Tokens per sec: 83464.20440530777, Loss: 2.2645187377929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9280 , TFLOPS: 102.07977076275799, Tokens per sec: 83412.0084573217, Loss: 2.2926132678985596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9281 , TFLOPS: 102.68474474723328, Tokens per sec: 83906.34827345227, Loss: 2.2855887413024902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9282 , TFLOPS: 102.26334041929748, Tokens per sec: 83562.00794918348, Loss: 2.258774757385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9283 , TFLOPS: 102.62881128373573, Tokens per sec: 83860.64360057296, Loss: 2.283435821533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9284 , TFLOPS: 100.7421974864066, Tokens per sec: 82319.04290101567, Loss: 2.273111343383789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9285 , TFLOPS: 102.74492354768753, Tokens per sec: 83955.52192044359, Loss: 2.2610456943511963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9286 , TFLOPS: 102.0833276925114, Tokens per sec: 83414.91491618672, Loss: 2.2679734230041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9287 , TFLOPS: 103.3009151664274, Tokens per sec: 84409.83698461346, Loss: 2.2827749252319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9288 , TFLOPS: 101.46048945911423, Tokens per sec: 82905.97776239541, Loss: 2.262770175933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9289 , TFLOPS: 100.86204110208492, Tokens per sec: 82416.97020443555, Loss: 2.2610764503479004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9290 , TFLOPS: 102.8570195375621, Tokens per sec: 84047.11843937756, Loss: 2.2671546936035156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9291 , TFLOPS: 102.82168507374823, Tokens per sec: 84018.24573940525, Loss: 2.251521348953247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9292 , TFLOPS: 100.80793880478491, Tokens per sec: 82372.76182459467, Loss: 2.2508578300476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9293 , TFLOPS: 102.88278017359103, Tokens per sec: 84068.16811821458, Loss: 2.2548134326934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9294 , TFLOPS: 102.66994573289647, Tokens per sec: 83894.25561787713, Loss: 2.2684624195098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9295 , TFLOPS: 101.00158184433367, Tokens per sec: 82530.99253702527, Loss: 2.266528367996216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9296 , TFLOPS: 102.23421843355796, Tokens per sec: 83538.21162496909, Loss: 2.2481377124786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9297 , TFLOPS: 102.60159962611067, Tokens per sec: 83838.40825463715, Loss: 2.2758002281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9298 , TFLOPS: 100.84266783721019, Tokens per sec: 82401.1398109942, Loss: 2.26953387260437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9299 , TFLOPS: 102.92405547339128, Tokens per sec: 84101.8952282021, Loss: 2.2710683345794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9300 , TFLOPS: 101.54661216445395, Tokens per sec: 82976.35084192416, Loss: 2.2491588592529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9301 , TFLOPS: 102.85803349070521, Tokens per sec: 84047.94696659228, Loss: 2.26304030418396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9302 , TFLOPS: 100.92658940334496, Tokens per sec: 82469.71428301623, Loss: 2.2551915645599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9303 , TFLOPS: 101.74258343097125, Tokens per sec: 83136.48400854465, Loss: 2.2609517574310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9304 , TFLOPS: 101.49497404652934, Tokens per sec: 82934.15600648454, Loss: 2.269184112548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9305 , TFLOPS: 100.85943300810685, Tokens per sec: 82414.83906370775, Loss: 2.2610647678375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9306 , TFLOPS: 102.12025914716476, Tokens per sec: 83445.0925584842, Loss: 2.281721830368042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9307 , TFLOPS: 100.72478415111216, Tokens per sec: 82304.81401648745, Loss: 2.2426865100860596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9308 , TFLOPS: 102.27399533485344, Tokens per sec: 83570.71435496616, Loss: 2.25838565826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9309 , TFLOPS: 101.71415421080505, Tokens per sec: 83113.25376091359, Loss: 2.26192307472229 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9310 , TFLOPS: 101.96743611431029, Tokens per sec: 83320.21692432398, Loss: 2.2827188968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9311 , TFLOPS: 101.9044909641025, Tokens per sec: 83268.78282173687, Loss: 2.253722906112671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9312 , TFLOPS: 102.0839897009947, Tokens per sec: 83415.45586036005, Loss: 2.2964439392089844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9313 , TFLOPS: 102.84862439374926, Tokens per sec: 84040.25855125788, Loss: 2.270265817642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9314 , TFLOPS: 102.42175615899554, Tokens per sec: 83691.45352807468, Loss: 2.2629363536834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9315 , TFLOPS: 101.488745533315, Tokens per sec: 82929.06652800091, Loss: 2.2614426612854004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9316 , TFLOPS: 102.07949403007457, Tokens per sec: 83411.78233192234, Loss: 2.2548978328704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9317 , TFLOPS: 100.58901097048782, Tokens per sec: 82193.8702554867, Loss: 2.2869369983673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9318 , TFLOPS: 102.81897241949503, Tokens per sec: 84016.02915978502, Loss: 2.2595880031585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9319 , TFLOPS: 102.72223658021285, Tokens per sec: 83936.9838153055, Loss: 2.263871669769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9320 , TFLOPS: 103.42851640589456, Tokens per sec: 84514.1032421302, Loss: 2.2465617656707764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9321 , TFLOPS: 100.8268277554999, Tokens per sec: 82388.19647246874, Loss: 2.26175856590271 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9322 , TFLOPS: 102.73096974611309, Tokens per sec: 83944.1199099743, Loss: 2.2462494373321533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9323 , TFLOPS: 102.09185319773339, Tokens per sec: 83421.88132596963, Loss: 2.2596452236175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9324 , TFLOPS: 100.7495131083804, Tokens per sec: 82325.02068405131, Loss: 2.273538112640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9325 , TFLOPS: 103.2759208675728, Tokens per sec: 84389.41349960887, Loss: 2.2822563648223877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9326 , TFLOPS: 102.26113584332079, Tokens per sec: 83560.20653340207, Loss: 2.258343458175659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9327 , TFLOPS: 101.45355510469204, Tokens per sec: 82900.31152289089, Loss: 2.2427265644073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9328 , TFLOPS: 102.17374345354875, Tokens per sec: 83488.7959620391, Loss: 2.250751256942749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9329 , TFLOPS: 103.32702123998578, Tokens per sec: 84431.16892934813, Loss: 2.286635637283325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9330 , TFLOPS: 100.88534034998958, Tokens per sec: 82436.00861967444, Loss: 2.282900094985962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9331 , TFLOPS: 102.30818064754509, Tokens per sec: 83598.64805397476, Loss: 2.268061637878418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9332 , TFLOPS: 101.64240891707537, Tokens per sec: 83054.62883451892, Loss: 2.2767937183380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9333 , TFLOPS: 101.25882954001827, Tokens per sec: 82741.19625131397, Loss: 2.2650258541107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9334 , TFLOPS: 100.98781783384236, Tokens per sec: 82519.74560973566, Loss: 2.2523269653320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9335 , TFLOPS: 102.78154538899888, Tokens per sec: 83985.44656970915, Loss: 2.263477087020874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9336 , TFLOPS: 101.38042123892967, Tokens per sec: 82840.55195854325, Loss: 2.2538576126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9337 , TFLOPS: 102.15936521668627, Tokens per sec: 83477.04713457002, Loss: 2.2658591270446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9338 , TFLOPS: 102.29565503612713, Tokens per sec: 83588.41305444721, Loss: 2.278102159500122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9339 , TFLOPS: 100.43136098080086, Tokens per sec: 82065.05039064171, Loss: 2.275987148284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9340 , TFLOPS: 102.05519098937069, Tokens per sec: 83391.92369174727, Loss: 2.2709505558013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9341 , TFLOPS: 101.53622302722835, Tokens per sec: 82967.86161045694, Loss: 2.2583820819854736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9342 , TFLOPS: 102.65813333518769, Tokens per sec: 83884.60340363112, Loss: 2.24979567527771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9343 , TFLOPS: 101.50657884245986, Tokens per sec: 82943.6385839734, Loss: 2.2446227073669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9344 , TFLOPS: 102.17352561780243, Tokens per sec: 83488.61796284319, Loss: 2.2694199085235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9345 , TFLOPS: 102.24696079239607, Tokens per sec: 83548.62373439322, Loss: 2.2584054470062256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9346 , TFLOPS: 102.46924144104166, Tokens per sec: 83730.25497441446, Loss: 2.2587051391601562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9347 , TFLOPS: 102.09627618285195, Tokens per sec: 83425.49546096778, Loss: 2.2587130069732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9348 , TFLOPS: 102.00833761078057, Tokens per sec: 83353.63858998803, Loss: 2.2609636783599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9349 , TFLOPS: 101.37502870307043, Tokens per sec: 82836.14557867644, Loss: 2.255141019821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9350 , TFLOPS: 103.23951601097376, Tokens per sec: 84359.66615413742, Loss: 2.2646279335021973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9351 , TFLOPS: 102.00329638057464, Tokens per sec: 83349.51927101395, Loss: 2.272212028503418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9352 , TFLOPS: 102.92790536221545, Tokens per sec: 84105.0410714653, Loss: 2.2708663940429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9353 , TFLOPS: 102.1453845548243, Tokens per sec: 83465.62317586836, Loss: 2.246943235397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9354 , TFLOPS: 101.34723262922802, Tokens per sec: 82813.43269120522, Loss: 2.2593495845794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9355 , TFLOPS: 101.51070325362177, Tokens per sec: 82947.00874650558, Loss: 2.2665436267852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9356 , TFLOPS: 101.62110922871109, Tokens per sec: 83037.22431085372, Loss: 2.257999897003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9357 , TFLOPS: 103.33119371149024, Tokens per sec: 84434.57836322341, Loss: 2.2636497020721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9358 , TFLOPS: 103.29300501585917, Tokens per sec: 84403.37339695878, Loss: 2.2889437675476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9359 , TFLOPS: 100.58960880140582, Tokens per sec: 82194.35875851933, Loss: 2.2681186199188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9360 , TFLOPS: 102.0549104327669, Tokens per sec: 83391.69444172415, Loss: 2.2756576538085938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9361 , TFLOPS: 102.67243712605811, Tokens per sec: 83896.29139936369, Loss: 2.2434072494506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9362 , TFLOPS: 100.88559761956905, Tokens per sec: 82436.21884127054, Loss: 2.275531053543091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9363 , TFLOPS: 102.89533630758919, Tokens per sec: 84078.42805852812, Loss: 2.266634702682495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9364 , TFLOPS: 101.04252938724454, Tokens per sec: 82564.45182842127, Loss: 2.2621071338653564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9365 , TFLOPS: 102.09918515726969, Tokens per sec: 83427.87245884817, Loss: 2.2570343017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9366 , TFLOPS: 100.78880491687654, Tokens per sec: 82357.12703223465, Loss: 2.252831220626831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9367 , TFLOPS: 103.49032389094792, Tokens per sec: 84564.60772923351, Loss: 2.281095504760742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9368 , TFLOPS: 100.43805718953905, Tokens per sec: 82070.52203517745, Loss: 2.2375235557556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9369 , TFLOPS: 102.74870501922005, Tokens per sec: 83958.61185817653, Loss: 2.2730228900909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9370 , TFLOPS: 101.63174382684755, Tokens per sec: 83045.91411474993, Loss: 2.260671615600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9371 , TFLOPS: 101.07584778391059, Tokens per sec: 82591.67714802893, Loss: 2.2631399631500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9372 , TFLOPS: 101.91749360046431, Tokens per sec: 83279.40761062576, Loss: 2.242461919784546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9373 , TFLOPS: 101.42743040250907, Tokens per sec: 82878.96435622752, Loss: 2.2609407901763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9374 , TFLOPS: 101.43615389898686, Tokens per sec: 82886.0925497623, Loss: 2.2646517753601074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9375 , TFLOPS: 102.00186867980916, Tokens per sec: 83348.35265996619, Loss: 2.2696011066436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9376 , TFLOPS: 102.21593057741727, Tokens per sec: 83523.26814694524, Loss: 2.2540249824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9377 , TFLOPS: 102.301028632111, Tokens per sec: 83592.8039580543, Loss: 2.2666802406311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9378 , TFLOPS: 101.61949889584096, Tokens per sec: 83035.90846641187, Loss: 2.276963233947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9379 , TFLOPS: 102.11476896264631, Tokens per sec: 83440.60638738425, Loss: 2.2551772594451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9380 , TFLOPS: 102.70946715956299, Tokens per sec: 83926.5495929784, Loss: 2.266234874725342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9381 , TFLOPS: 100.59150316268848, Tokens per sec: 82195.9066898886, Loss: 2.2957119941711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9382 , TFLOPS: 103.4799256848651, Tokens per sec: 84556.1110873696, Loss: 2.258636951446533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9383 , TFLOPS: 102.11543839924715, Tokens per sec: 83441.15340126364, Loss: 2.2713282108306885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9384 , TFLOPS: 102.8046098232284, Tokens per sec: 84004.29311264945, Loss: 2.2531683444976807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9385 , TFLOPS: 102.66679696833233, Tokens per sec: 83891.68268129537, Loss: 2.2407827377319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9386 , TFLOPS: 101.89455265598839, Tokens per sec: 83260.66197434202, Loss: 2.263256311416626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9387 , TFLOPS: 100.83889178813186, Tokens per sec: 82398.05430408815, Loss: 2.250047206878662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9388 , TFLOPS: 103.38699180741423, Tokens per sec: 84480.17242377368, Loss: 2.278916358947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9389 , TFLOPS: 102.0488046733027, Tokens per sec: 83386.70526848972, Loss: 2.2577712535858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9390 , TFLOPS: 102.77753698215746, Tokens per sec: 83982.17119729344, Loss: 2.284672737121582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9391 , TFLOPS: 102.13177253282684, Tokens per sec: 83454.5004422896, Loss: 2.2421600818634033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9392 , TFLOPS: 101.76424923178304, Tokens per sec: 83154.18769211529, Loss: 2.2776005268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9393 , TFLOPS: 101.39397846320185, Tokens per sec: 82851.62991549005, Loss: 2.2698757648468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9394 , TFLOPS: 102.33419116008636, Tokens per sec: 83619.90191334239, Loss: 2.249570369720459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9395 , TFLOPS: 102.1617653809152, Tokens per sec: 83479.0083705472, Loss: 2.266470193862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9396 , TFLOPS: 103.43274061464055, Tokens per sec: 84517.55495183736, Loss: 2.257317066192627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9397 , TFLOPS: 101.43571071813342, Tokens per sec: 82885.73041527845, Loss: 2.281355381011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9398 , TFLOPS: 101.39207867633324, Tokens per sec: 82850.07755073484, Loss: 2.278529167175293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9399 , TFLOPS: 103.36760441724743, Tokens per sec: 84464.33048819256, Loss: 2.2828269004821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9400 , TFLOPS: 100.24240267925522, Tokens per sec: 81910.64769822937, Loss: 2.270585060119629 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/model.pt +[rank0]:[2024-08-28 15:42:44,315] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007724005001364276, 'preprocessing_with_comm': 0.0015144360077101737, 'state_converting': 3.3282328580098692, : 3.339092788984999}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400, took 15.88s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007400 + eval ppl=8.406756401062012, eval loss=2.129035711288452 +------------------------------------------------------------------ +iteration: 9401 , TFLOPS: 101.9859523495456, Tokens per sec: 83335.34701678505, Loss: 2.249020576477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9402 , TFLOPS: 102.22750236643003, Tokens per sec: 83532.72375363193, Loss: 2.269810199737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9403 , TFLOPS: 102.94153792221506, Tokens per sec: 84116.18058717428, Loss: 2.2634122371673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9404 , TFLOPS: 101.71952051392432, Tokens per sec: 83117.6387053333, Loss: 2.258298635482788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9405 , TFLOPS: 101.11550888804364, Tokens per sec: 82624.08525718369, Loss: 2.2636351585388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9406 , TFLOPS: 102.79546349643728, Tokens per sec: 83996.81941358102, Loss: 2.2540690898895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9407 , TFLOPS: 103.28333053393736, Tokens per sec: 84395.46812873713, Loss: 2.2674777507781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9408 , TFLOPS: 102.83520598220413, Tokens per sec: 84029.29402176371, Loss: 2.2766542434692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9409 , TFLOPS: 102.85078634937445, Tokens per sec: 84042.02514085291, Loss: 2.23652982711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9410 , TFLOPS: 101.34906499745811, Tokens per sec: 82814.92996645538, Loss: 2.258439064025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9411 , TFLOPS: 103.32693170578253, Tokens per sec: 84431.09576864584, Loss: 2.2599425315856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9412 , TFLOPS: 101.37128249933302, Tokens per sec: 82833.08445916674, Loss: 2.272592306137085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9413 , TFLOPS: 102.50940324119298, Tokens per sec: 83763.07221517483, Loss: 2.269251823425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9414 , TFLOPS: 101.26855925705591, Tokens per sec: 82749.14665357061, Loss: 2.2529215812683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9415 , TFLOPS: 101.85357467768436, Tokens per sec: 83227.1778134028, Loss: 2.2717390060424805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9416 , TFLOPS: 101.8941211733435, Tokens per sec: 83260.30939876554, Loss: 2.2886481285095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9417 , TFLOPS: 101.47526332977573, Tokens per sec: 82918.04987242619, Loss: 2.2473325729370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9418 , TFLOPS: 102.11175133370875, Tokens per sec: 83438.1406050987, Loss: 2.2575795650482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9419 , TFLOPS: 102.18852438025397, Tokens per sec: 83500.87383774483, Loss: 2.2467212677001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9420 , TFLOPS: 100.09607098598157, Tokens per sec: 81791.07630474256, Loss: 2.244900941848755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9421 , TFLOPS: 102.70795356128826, Tokens per sec: 83925.31279286463, Loss: 2.2552452087402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9422 , TFLOPS: 101.54649571664973, Tokens per sec: 82976.2556894257, Loss: 2.26507306098938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9423 , TFLOPS: 101.70790040475754, Tokens per sec: 83108.1436150049, Loss: 2.2820346355438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9424 , TFLOPS: 100.30780876398875, Tokens per sec: 81964.09269376752, Loss: 2.279733657836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9425 , TFLOPS: 100.9292369951179, Tokens per sec: 82471.87769840896, Loss: 2.2681820392608643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9426 , TFLOPS: 101.52685346857388, Tokens per sec: 82960.20550288647, Loss: 2.2536144256591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9427 , TFLOPS: 101.20777187841469, Tokens per sec: 82699.47572167653, Loss: 2.2649388313293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9428 , TFLOPS: 101.10076208497719, Tokens per sec: 82612.03526478162, Loss: 2.261725425720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9429 , TFLOPS: 101.74646342217595, Tokens per sec: 83139.65444923785, Loss: 2.2684788703918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9430 , TFLOPS: 100.8510435211924, Tokens per sec: 82407.98379798535, Loss: 2.260265350341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9431 , TFLOPS: 101.5861876397985, Tokens per sec: 83008.68898159155, Loss: 2.272745370864868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9432 , TFLOPS: 102.82280255891965, Tokens per sec: 84019.15886530544, Loss: 2.251549243927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9433 , TFLOPS: 101.5490847129601, Tokens per sec: 82978.37122495759, Loss: 2.261749505996704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9434 , TFLOPS: 102.87660962684294, Tokens per sec: 84063.1260056227, Loss: 2.2701268196105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9435 , TFLOPS: 101.69764343512213, Tokens per sec: 83099.76238107792, Loss: 2.2563323974609375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9436 , TFLOPS: 101.57734528855846, Tokens per sec: 83001.46366877096, Loss: 2.280118942260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9437 , TFLOPS: 101.39983338645575, Tokens per sec: 82856.41412399996, Loss: 2.2706916332244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9438 , TFLOPS: 101.73281564115726, Tokens per sec: 83128.50249604235, Loss: 2.258451461791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9439 , TFLOPS: 102.27900761418235, Tokens per sec: 83574.81001743337, Loss: 2.2544851303100586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9440 , TFLOPS: 102.29841618234272, Tokens per sec: 83590.6692580982, Loss: 2.25393009185791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9441 , TFLOPS: 101.03241255019263, Tokens per sec: 82556.18510043561, Loss: 2.2779743671417236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9442 , TFLOPS: 101.05213651017102, Tokens per sec: 82572.30205587384, Loss: 2.2562286853790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9443 , TFLOPS: 102.97120477500944, Tokens per sec: 84140.4221362853, Loss: 2.2637741565704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9444 , TFLOPS: 102.78485080084789, Tokens per sec: 83988.14750681985, Loss: 2.259749174118042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9445 , TFLOPS: 102.76959935006771, Tokens per sec: 83975.68515378007, Loss: 2.2713217735290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9446 , TFLOPS: 102.36455546062169, Tokens per sec: 83644.71336495643, Loss: 2.2773966789245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9447 , TFLOPS: 102.36098597998178, Tokens per sec: 83641.79665043895, Loss: 2.24973726272583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9448 , TFLOPS: 101.39738129781874, Tokens per sec: 82854.41045925226, Loss: 2.2457356452941895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9449 , TFLOPS: 102.57664285147948, Tokens per sec: 83818.01543164147, Loss: 2.270249128341675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9450 , TFLOPS: 101.47495428445535, Tokens per sec: 82917.79734353938, Loss: 2.2691118717193604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9451 , TFLOPS: 102.02041411535178, Tokens per sec: 83363.50661275005, Loss: 2.245638370513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9452 , TFLOPS: 101.63827414405216, Tokens per sec: 83051.25020504312, Loss: 2.2819976806640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9453 , TFLOPS: 102.24627716159802, Tokens per sec: 83548.06512206991, Loss: 2.24784255027771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9454 , TFLOPS: 102.1904560557611, Tokens per sec: 83502.45225952746, Loss: 2.2566399574279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9455 , TFLOPS: 100.2893724780068, Tokens per sec: 81949.02792989946, Loss: 2.260498523712158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9456 , TFLOPS: 102.51508126266151, Tokens per sec: 83767.71187268203, Loss: 2.250669002532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9457 , TFLOPS: 101.48229487523332, Tokens per sec: 82923.79552922767, Loss: 2.2870495319366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9458 , TFLOPS: 98.32942056174991, Tokens per sec: 80347.50076547515, Loss: 2.2631070613861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9459 , TFLOPS: 101.21724192544754, Tokens per sec: 82707.21394088777, Loss: 2.254384756088257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9460 , TFLOPS: 102.85110530746957, Tokens per sec: 84042.28576972314, Loss: 2.2431678771972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9461 , TFLOPS: 100.38701379552079, Tokens per sec: 82028.81316395126, Loss: 2.254927396774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9462 , TFLOPS: 101.58958543580108, Tokens per sec: 83011.46540817231, Loss: 2.257885456085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9463 , TFLOPS: 101.71514955182276, Tokens per sec: 83114.06707968154, Loss: 2.2533979415893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9464 , TFLOPS: 102.13492949442607, Tokens per sec: 83457.08007687975, Loss: 2.2509877681732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9465 , TFLOPS: 101.47576676279166, Tokens per sec: 82918.46124050303, Loss: 2.2647221088409424 diff --git a/wandb/run-20240826_221726-7jzdp89j/files/requirements.txt b/wandb/run-20240826_221726-7jzdp89j/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240826_221726-7jzdp89j/files/wandb-metadata.json b/wandb/run-20240826_221726-7jzdp89j/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b1ce61eb5dc4b3bf66490840cdbf75dd422086b6 --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-26T13:17:27.721371", + "startedAt": "2024-08-26T13:17:26.981295", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-5", + "--min-lr", + "3.5e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5", + "--fsdp-activation-checkpointing", + "--num-workers", + "4", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-1.5B-3.5e-5", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-26-22:17:00" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0340000000015, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.034, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.4449996948242 + } +} diff --git a/wandb/run-20240826_221726-7jzdp89j/files/wandb-summary.json b/wandb/run-20240826_221726-7jzdp89j/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ec7f2544230c3f40ba392c51630f24f2e73d08cf --- /dev/null +++ b/wandb/run-20240826_221726-7jzdp89j/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.2509877681732178, "training/perplexity": 9.497112149903268, "utils/batch_size": 16, "utils/global_batch_size": 1280, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 10, "utils/iteration": 9464, "optimizer/lr": 2.433893531560898e-05, "optimizer/variance_l2": 0.0010765908338857258, "optimizer/variance_sqrt_l2": 0.17256405729442909, "optimizer/momentum_l2": 0.037983187208901335, "optimizer/weight_l2": 1232.9854013734307, "optimizer/variance_l1": 0.02979755401611328, "optimizer/variance_sqrt_l1": 952.25, "optimizer/momentum_l1": 172.59375, "optimizer/weight_l1": 3158016.0, "optimizer/variance_abs_max": 0.00066375732421875, "optimizer/variance_sqrt_abs_max": 0.0257568359375, "optimizer/momentum_abs_max": 0.0057373046875, "optimizer/weight_abs_max": 432.0, "stats/1_iteration_time": 15.720655440993141, "stats/tokens_per_sec": 83457.08007687975, "stats/tokens_per_sec_per_gpu": 10432.135009609969, "stats/tflops": 102.13492949442607, "_timestamp": 1724828384.1428092, "_runtime": 150137.14689826965, "_step": 9464, "evaluation/val_loss": 2.129035711288452, "evaluation/val_ppl": 8.406756401062012} \ No newline at end of file diff --git a/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml b/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94ecde22d052381ed500288d826f28dd30bc919a --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml @@ -0,0 +1,342 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: FULL_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '1754785366' + - /project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document + - '28623823675' + - /project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document +valid_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +test_data_path: + desc: null + value: + - '1205770' + - /work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-1.5B +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-28-21:29:36 +wandb_project: + desc: null + value: yans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5 +save: + desc: null + value: /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5 +base_model: + desc: null + value: /share/pretrained_lm/Qwen/Qwen2-1.5B +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 3.5e-05 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 23178 +lr_warmup_iters: + desc: null + value: 500 +min_lr: + desc: null + value: 3.5e-06 +train_iters: + desc: null + value: 23178 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 1280 +micro_batch_size: + desc: null + value: 16 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/yans-baseline-qwen2-1.5B-3.5e-5 +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 8 +padded_vocab_size: + desc: null + value: 151680 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1724848216.390277 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: Qwen2ForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 1536 +model_type: + desc: null + value: qwen2 +max_position_embeddings: + desc: null + value: 1024 +num_attention_heads: + desc: null + value: 12 +num_hidden_layers: + desc: null + value: 28 diff --git a/wandb/run-20240828_213016-pkqwjpaw/files/output.log b/wandb/run-20240828_213016-pkqwjpaw/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..656388be53f6145c4bc4fcc1159008ff23eabd7e --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/files/output.log @@ -0,0 +1,13615 @@ +Created Hugging Face repository with ID koichi12/yans-baseline-qwen2-1.5B-3.5e-5. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +Loading model state dict from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/model.pt +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Loaded model state dict from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/model.pt +--> Model /share/pretrained_lm/Qwen/Qwen2-1.5B +--> /share/pretrained_lm/Qwen/Qwen2-1.5B has 1543.714304 Million params +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 29667840 + validation: 1484800 + test: 12800 +> building train, validation, and test datasets for GPT ... +Let split = None +Unable to save the indexes because path_to_cache is None +> finished creating GPT datasets ... +Loading optimizer state dict from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/optimizer.pt +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Loaded optimizer state dict from /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400/optimizer.pt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): Qwen2ForCausalLM( + (model): Qwen2Model( + (embed_tokens): Embedding(151936, 1536) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): Qwen2DecoderLayer( + (self_attn): Qwen2FlashAttention2( + (q_proj): Linear(in_features=1536, out_features=1536, bias=True) + (k_proj): Linear(in_features=1536, out_features=256, bias=True) + (v_proj): Linear(in_features=1536, out_features=256, bias=True) + (o_proj): Linear(in_features=1536, out_features=1536, bias=False) + (rotary_emb): Qwen2RotaryEmbedding() + ) + (mlp): Qwen2MLP( + (gate_proj): Linear(in_features=1536, out_features=8960, bias=False) + (up_proj): Linear(in_features=1536, out_features=8960, bias=False) + (down_proj): Linear(in_features=8960, out_features=1536, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): Qwen2RMSNorm() + (post_attention_layernorm): Qwen2RMSNorm() + ) + ) + ) + ) + (norm): Qwen2RMSNorm() + ) + (lm_head): Linear(in_features=1536, out_features=151936, bias=False) + ) +) +model config: Qwen2Config { + "_name_or_path": "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "architectures": [ + "Qwen2ForCausalLM" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151643, + "hidden_act": "silu", + "hidden_size": 1536, + "initializer_range": 0.02, + "intermediate_size": 8960, + "label_smoothing": 0.0, + "max_position_embeddings": 1024, + "max_window_layers": 28, + "model_type": "qwen2", + "num_attention_heads": 12, + "num_hidden_layers": 28, + "num_key_value_heads": 2, + "rms_norm_eps": 1e-06, + "rope_theta": 1000000.0, + "sliding_window": 131072, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "use_sliding_window": false, + "vocab_size": 151936 +} +[rank0]:[2024-08-28 21:31:23,444] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _flatten_optim_state_dict() profiling: defaultdict(, {}) +------------------------------------------------------------------ +iteration: 9401 , TFLOPS: 50.558358410020766, Tokens per sec: 41312.536144757054, Loss: 2.249020576477051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9402 , TFLOPS: 101.10635175156914, Tokens per sec: 82616.60272524471, Loss: 2.2698159217834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9403 , TFLOPS: 99.72636049394625, Tokens per sec: 81488.97634450576, Loss: 2.26344633102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9404 , TFLOPS: 100.46362543998677, Tokens per sec: 82091.41451080359, Loss: 2.258265495300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9405 , TFLOPS: 99.84221699744504, Tokens per sec: 81583.64567592635, Loss: 2.2635765075683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9406 , TFLOPS: 100.51023004508683, Tokens per sec: 82129.49633334011, Loss: 2.2541303634643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9407 , TFLOPS: 100.5702950511043, Tokens per sec: 82178.5769959678, Loss: 2.267426013946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9408 , TFLOPS: 101.31832304775472, Tokens per sec: 82789.80992798455, Loss: 2.2766823768615723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9409 , TFLOPS: 100.61228513245031, Tokens per sec: 82212.88817235648, Loss: 2.2365787029266357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9410 , TFLOPS: 100.66787037602487, Tokens per sec: 82258.30830577272, Loss: 2.2584078311920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9411 , TFLOPS: 101.27607521286942, Tokens per sec: 82755.28813454375, Loss: 2.259920597076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9412 , TFLOPS: 100.69131400107891, Tokens per sec: 82277.46469529698, Loss: 2.272559404373169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9413 , TFLOPS: 101.25657873292552, Tokens per sec: 82739.35705889754, Loss: 2.2693026065826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9414 , TFLOPS: 99.76363045756011, Tokens per sec: 81519.43059118888, Loss: 2.2529239654541016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9415 , TFLOPS: 101.10962310673733, Tokens per sec: 82619.27583376481, Loss: 2.271780252456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9416 , TFLOPS: 101.27962548084322, Tokens per sec: 82758.18914989717, Loss: 2.288644313812256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9417 , TFLOPS: 101.28079703759227, Tokens per sec: 82759.14645907537, Loss: 2.2473225593566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9418 , TFLOPS: 100.46335644491101, Tokens per sec: 82091.19470800264, Loss: 2.257545232772827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9419 , TFLOPS: 101.30875664912962, Tokens per sec: 82781.99297740698, Loss: 2.2467081546783447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9420 , TFLOPS: 100.47004528422515, Tokens per sec: 82096.66033079226, Loss: 2.2448668479919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9421 , TFLOPS: 99.83869009293025, Tokens per sec: 81580.76375145672, Loss: 2.2552595138549805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9422 , TFLOPS: 101.32267380067229, Tokens per sec: 82793.36504019184, Loss: 2.2650132179260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9423 , TFLOPS: 101.13598661149675, Tokens per sec: 82640.81813216073, Loss: 2.282076597213745 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9424 , TFLOPS: 99.84932179379, Tokens per sec: 81589.45118791354, Loss: 2.279764413833618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9425 , TFLOPS: 101.21825970467897, Tokens per sec: 82708.04559449776, Loss: 2.268199920654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9426 , TFLOPS: 101.1679860948348, Tokens per sec: 82666.96573373623, Loss: 2.253612518310547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9427 , TFLOPS: 98.98097203487725, Tokens per sec: 80879.90024659454, Loss: 2.2648863792419434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9428 , TFLOPS: 101.27057411701459, Tokens per sec: 82750.79304750997, Loss: 2.2617180347442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9429 , TFLOPS: 100.55399766672208, Tokens per sec: 82165.25998365709, Loss: 2.268479824066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9430 , TFLOPS: 100.11769585373462, Tokens per sec: 81808.7465408572, Loss: 2.2602922916412354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9431 , TFLOPS: 100.47181420775803, Tokens per sec: 82098.10576374712, Loss: 2.2728207111358643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9432 , TFLOPS: 100.64103179229573, Tokens per sec: 82236.37780812103, Loss: 2.2514965534210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9433 , TFLOPS: 99.72909297049308, Tokens per sec: 81491.20912143237, Loss: 2.2617909908294678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9434 , TFLOPS: 100.39197888774027, Tokens per sec: 82032.87026861664, Loss: 2.2701032161712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9435 , TFLOPS: 100.5934613169702, Tokens per sec: 82197.50674815958, Loss: 2.2563657760620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9436 , TFLOPS: 99.90033942758754, Tokens per sec: 81631.13900980024, Loss: 2.2802083492279053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9437 , TFLOPS: 99.82176045859366, Tokens per sec: 81566.93011143274, Loss: 2.270636796951294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9438 , TFLOPS: 101.30353627732788, Tokens per sec: 82777.72727722346, Loss: 2.258432388305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9439 , TFLOPS: 99.70427361127723, Tokens per sec: 81470.92858410996, Loss: 2.254506826400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9440 , TFLOPS: 101.31266299092626, Tokens per sec: 82785.18494984719, Loss: 2.253946304321289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9441 , TFLOPS: 100.62543490388698, Tokens per sec: 82223.63318910262, Loss: 2.2779619693756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9442 , TFLOPS: 99.84186308576353, Tokens per sec: 81583.3564855809, Loss: 2.256263256072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9443 , TFLOPS: 101.26944928293413, Tokens per sec: 82749.87391662701, Loss: 2.263815402984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9444 , TFLOPS: 99.89268284394237, Tokens per sec: 81624.88261820577, Loss: 2.2597408294677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9445 , TFLOPS: 100.22493757222301, Tokens per sec: 81896.3765096815, Loss: 2.2713215351104736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9446 , TFLOPS: 101.24354105960951, Tokens per sec: 82728.70364041146, Loss: 2.277419090270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9447 , TFLOPS: 100.46893955868948, Tokens per sec: 82095.75681399317, Loss: 2.2497408390045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9448 , TFLOPS: 101.34247548494095, Tokens per sec: 82809.54550614847, Loss: 2.2457666397094727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9449 , TFLOPS: 100.663275611168, Tokens per sec: 82254.5538051284, Loss: 2.2702362537384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9450 , TFLOPS: 99.87640284979663, Tokens per sec: 81611.57981590496, Loss: 2.2690393924713135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9451 , TFLOPS: 101.33711710265237, Tokens per sec: 82805.16703404344, Loss: 2.245673418045044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9452 , TFLOPS: 99.01118923616977, Tokens per sec: 80904.59149963061, Loss: 2.2819414138793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9453 , TFLOPS: 100.58574104199867, Tokens per sec: 82191.19831274277, Loss: 2.2478787899017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9454 , TFLOPS: 100.62417365246601, Tokens per sec: 82222.60258809905, Loss: 2.2566163539886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9455 , TFLOPS: 101.36842343063417, Tokens per sec: 82830.74824053394, Loss: 2.260529041290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9456 , TFLOPS: 99.97022149622808, Tokens per sec: 81688.24144701092, Loss: 2.250692844390869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9457 , TFLOPS: 101.32406172596973, Tokens per sec: 82794.49914968076, Loss: 2.286954641342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9458 , TFLOPS: 100.62930557155045, Tokens per sec: 82226.79601129028, Loss: 2.26312255859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9459 , TFLOPS: 100.03384773744597, Tokens per sec: 81740.2320865952, Loss: 2.254389524459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9460 , TFLOPS: 101.31465870142596, Tokens per sec: 82786.81569627066, Loss: 2.243252754211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9461 , TFLOPS: 100.60744322338232, Tokens per sec: 82208.9317238152, Loss: 2.254843235015869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9462 , TFLOPS: 100.09106437051294, Tokens per sec: 81786.98527036139, Loss: 2.257821559906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9463 , TFLOPS: 99.77535586660726, Tokens per sec: 81529.01172476017, Loss: 2.2535033226013184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9464 , TFLOPS: 101.21501908779204, Tokens per sec: 82705.39760301854, Loss: 2.250969171524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9465 , TFLOPS: 99.31785206090704, Tokens per sec: 81155.17358792684, Loss: 2.264708995819092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9466 , TFLOPS: 100.42923573256381, Tokens per sec: 82063.31379559847, Loss: 2.264486312866211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9467 , TFLOPS: 100.679052252532, Tokens per sec: 82267.44530491381, Loss: 2.2689337730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9468 , TFLOPS: 99.95872942422773, Tokens per sec: 81678.85097914717, Loss: 2.2609939575195312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9469 , TFLOPS: 99.19938019767898, Tokens per sec: 81058.36717874612, Loss: 2.236569881439209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9470 , TFLOPS: 101.26983128491136, Tokens per sec: 82750.18606027622, Loss: 2.260436534881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9471 , TFLOPS: 100.08055912209696, Tokens per sec: 81778.40116145155, Loss: 2.2695958614349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9472 , TFLOPS: 101.2545808784584, Tokens per sec: 82737.72456058311, Loss: 2.280735969543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9473 , TFLOPS: 100.13617376745417, Tokens per sec: 81823.84531982125, Loss: 2.231240749359131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9474 , TFLOPS: 100.09821924319486, Tokens per sec: 81792.8317010126, Loss: 2.23697829246521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9475 , TFLOPS: 100.59311381925833, Tokens per sec: 82197.22279883381, Loss: 2.246732473373413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9476 , TFLOPS: 100.60501171473575, Tokens per sec: 82206.9448755075, Loss: 2.2681171894073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9477 , TFLOPS: 98.7144808764377, Tokens per sec: 80662.14346094088, Loss: 2.2729482650756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9478 , TFLOPS: 100.63862399864766, Tokens per sec: 82234.41033794907, Loss: 2.251021146774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9479 , TFLOPS: 100.78542869171613, Tokens per sec: 82354.36823174432, Loss: 2.248718500137329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9480 , TFLOPS: 100.74825982022794, Tokens per sec: 82323.99659003945, Loss: 2.2664051055908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9481 , TFLOPS: 100.46994407975734, Tokens per sec: 82096.57763401639, Loss: 2.265777111053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9482 , TFLOPS: 99.950234577181, Tokens per sec: 81671.90962094849, Loss: 2.269728183746338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9483 , TFLOPS: 99.77539825700102, Tokens per sec: 81529.04636304213, Loss: 2.2663512229919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9484 , TFLOPS: 100.6449067133824, Tokens per sec: 82239.5441058903, Loss: 2.276050329208374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9485 , TFLOPS: 100.57233216409415, Tokens per sec: 82180.24157343175, Loss: 2.2600741386413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9486 , TFLOPS: 100.65925470851819, Tokens per sec: 82251.26822206599, Loss: 2.271578311920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9487 , TFLOPS: 101.38602830711105, Tokens per sec: 82845.13363829303, Loss: 2.22998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9488 , TFLOPS: 98.65480205895001, Tokens per sec: 80613.37836290224, Loss: 2.2765965461730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9489 , TFLOPS: 101.22007948069158, Tokens per sec: 82709.53258032286, Loss: 2.255218505859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9490 , TFLOPS: 99.23892669387618, Tokens per sec: 81090.68163880623, Loss: 2.2534258365631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9491 , TFLOPS: 99.32012723379266, Tokens per sec: 81157.03269025985, Loss: 2.272245168685913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9492 , TFLOPS: 100.57981589359672, Tokens per sec: 82186.35672145666, Loss: 2.2493505477905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9493 , TFLOPS: 100.56225520117124, Tokens per sec: 82172.00742762335, Loss: 2.2661266326904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9494 , TFLOPS: 98.63945996109042, Tokens per sec: 80600.84193980045, Loss: 2.2765989303588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9495 , TFLOPS: 99.88209356592563, Tokens per sec: 81616.22984654603, Loss: 2.255153179168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9496 , TFLOPS: 100.63472988850505, Tokens per sec: 82231.22836030822, Loss: 2.2576169967651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9497 , TFLOPS: 99.18746388399961, Tokens per sec: 81048.6300520855, Loss: 2.2812957763671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9498 , TFLOPS: 99.97832636325306, Tokens per sec: 81694.86414249486, Loss: 2.2672841548919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9499 , TFLOPS: 100.1509733264166, Tokens per sec: 81835.938420424, Loss: 2.266951084136963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9500 , TFLOPS: 99.46173787753041, Tokens per sec: 81272.74639264023, Loss: 2.2552525997161865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9501 , TFLOPS: 99.36624526737917, Tokens per sec: 81194.71692268716, Loss: 2.2567527294158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9502 , TFLOPS: 101.31573191872823, Tokens per sec: 82787.69264975541, Loss: 2.2457900047302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9503 , TFLOPS: 98.84714302184405, Tokens per sec: 80770.54511497983, Loss: 2.263293981552124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9504 , TFLOPS: 100.02538771904463, Tokens per sec: 81733.31918777923, Loss: 2.2544894218444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9505 , TFLOPS: 100.04706957789487, Tokens per sec: 81751.03599278646, Loss: 2.272962808609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9506 , TFLOPS: 99.58399866766878, Tokens per sec: 81372.64883153501, Loss: 2.268475294113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9507 , TFLOPS: 100.20676470260683, Tokens per sec: 81881.52699010684, Loss: 2.271458625793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9508 , TFLOPS: 100.75726482973153, Tokens per sec: 82331.35481511439, Loss: 2.2726783752441406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9509 , TFLOPS: 99.49968490528276, Tokens per sec: 81303.75388586007, Loss: 2.236304998397827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9510 , TFLOPS: 100.55668302282038, Tokens per sec: 82167.45425725222, Loss: 2.26448917388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9511 , TFLOPS: 99.31267495126838, Tokens per sec: 81150.94323837043, Loss: 2.2611944675445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9512 , TFLOPS: 100.72638003372185, Tokens per sec: 82306.11805324952, Loss: 2.239794969558716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9513 , TFLOPS: 100.63285952491614, Tokens per sec: 82229.70003807229, Loss: 2.2792365550994873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9514 , TFLOPS: 100.09255376592635, Tokens per sec: 81788.20229369415, Loss: 2.2646069526672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9515 , TFLOPS: 99.00505837812551, Tokens per sec: 80899.58181770032, Loss: 2.26296329498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9516 , TFLOPS: 100.03849292449814, Tokens per sec: 81744.0277885133, Loss: 2.2583909034729004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9517 , TFLOPS: 100.12625872708048, Tokens per sec: 81815.74348509609, Loss: 2.2886319160461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9518 , TFLOPS: 100.65662053018573, Tokens per sec: 82249.11576714076, Loss: 2.2446446418762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9519 , TFLOPS: 100.48828022966647, Tokens per sec: 82111.56057412163, Loss: 2.249565601348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9520 , TFLOPS: 98.2447368639831, Tokens per sec: 80278.30353607758, Loss: 2.262728214263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9521 , TFLOPS: 101.29100806668012, Tokens per sec: 82767.49015379832, Loss: 2.278151750564575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9522 , TFLOPS: 99.95325322080913, Tokens per sec: 81674.37623236382, Loss: 2.250908613204956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9523 , TFLOPS: 98.8929816101247, Tokens per sec: 80808.00100545419, Loss: 2.261634111404419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9524 , TFLOPS: 101.36588740335075, Tokens per sec: 82828.6759873575, Loss: 2.2750020027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9525 , TFLOPS: 99.98777357401056, Tokens per sec: 81702.58370159766, Loss: 2.2918622493743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9526 , TFLOPS: 99.23919681453505, Tokens per sec: 81090.90236135012, Loss: 2.2508702278137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9527 , TFLOPS: 99.91440303283942, Tokens per sec: 81642.63074368096, Loss: 2.2500104904174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9528 , TFLOPS: 99.87831673558703, Tokens per sec: 81613.14370125209, Loss: 2.2719950675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9529 , TFLOPS: 98.77702827399114, Tokens per sec: 80713.25254959502, Loss: 2.280604600906372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9530 , TFLOPS: 100.075731010309, Tokens per sec: 81774.45598702296, Loss: 2.266831159591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9531 , TFLOPS: 99.6429649596046, Tokens per sec: 81420.83170660316, Loss: 2.2605252265930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9532 , TFLOPS: 98.7783801612008, Tokens per sec: 80714.35721143472, Loss: 2.265434741973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9533 , TFLOPS: 98.2858683186445, Tokens per sec: 80311.91310649925, Loss: 2.252142906188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9534 , TFLOPS: 101.16385818325291, Tokens per sec: 82663.59271092077, Loss: 2.251504898071289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9535 , TFLOPS: 98.79118134116106, Tokens per sec: 80724.81738511194, Loss: 2.252718925476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9536 , TFLOPS: 99.02578411880356, Tokens per sec: 80916.51735393638, Loss: 2.2773892879486084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9537 , TFLOPS: 100.07694423069965, Tokens per sec: 81775.44734063522, Loss: 2.26643443107605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9538 , TFLOPS: 99.018246093314, Tokens per sec: 80910.35783926283, Loss: 2.2749552726745605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9539 , TFLOPS: 99.57480827443638, Tokens per sec: 81365.13912464306, Loss: 2.2838518619537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9540 , TFLOPS: 100.78990686886802, Tokens per sec: 82358.02746557366, Loss: 2.254406452178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9541 , TFLOPS: 99.306992020416, Tokens per sec: 81146.2995692793, Loss: 2.262495756149292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9542 , TFLOPS: 101.22036396539833, Tokens per sec: 82709.76504010003, Loss: 2.2786803245544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9543 , TFLOPS: 100.1217938887931, Tokens per sec: 81812.09515079629, Loss: 2.257256507873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9544 , TFLOPS: 101.20050303126368, Tokens per sec: 82693.53615955268, Loss: 2.2297885417938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9545 , TFLOPS: 100.12144190056169, Tokens per sec: 81811.80753215142, Loss: 2.293246030807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9546 , TFLOPS: 99.81959117386764, Tokens per sec: 81565.15753304075, Loss: 2.261958599090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9547 , TFLOPS: 99.48608289593633, Tokens per sec: 81292.63933387623, Loss: 2.259406328201294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9548 , TFLOPS: 100.01105547427936, Tokens per sec: 81721.60794163673, Loss: 2.269437074661255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9549 , TFLOPS: 99.47520416668833, Tokens per sec: 81283.75004416428, Loss: 2.2805862426757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9550 , TFLOPS: 100.07299311548776, Tokens per sec: 81772.21878268484, Loss: 2.27036190032959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9551 , TFLOPS: 101.32193870148423, Tokens per sec: 82792.76437172212, Loss: 2.2649178504943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9552 , TFLOPS: 97.02784045036313, Tokens per sec: 79283.9461508082, Loss: 2.2578253746032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9553 , TFLOPS: 101.22795400501174, Tokens per sec: 82715.96705685314, Loss: 2.257164478302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9554 , TFLOPS: 101.22209707668789, Tokens per sec: 82711.18120994898, Loss: 2.2673075199127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9555 , TFLOPS: 98.78505533232446, Tokens per sec: 80719.81166559405, Loss: 2.2667949199676514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9556 , TFLOPS: 101.18712578192897, Tokens per sec: 82682.60526476025, Loss: 2.253481388092041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9557 , TFLOPS: 100.0120690877963, Tokens per sec: 81722.43619133414, Loss: 2.267319917678833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9558 , TFLOPS: 99.35961723325197, Tokens per sec: 81189.30098538105, Loss: 2.271031141281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9559 , TFLOPS: 100.75350147066527, Tokens per sec: 82328.27967754388, Loss: 2.256753444671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9560 , TFLOPS: 100.04310835431204, Tokens per sec: 81747.79917502595, Loss: 2.246316432952881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9561 , TFLOPS: 98.11547704885585, Tokens per sec: 80172.68201369344, Loss: 2.228111743927002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9562 , TFLOPS: 100.7628484917939, Tokens per sec: 82335.91736912182, Loss: 2.283181667327881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9563 , TFLOPS: 99.52524989842836, Tokens per sec: 81324.64369986081, Loss: 2.251312494277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9564 , TFLOPS: 98.52212422691676, Tokens per sec: 80504.96389091664, Loss: 2.2590317726135254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9565 , TFLOPS: 98.81142318866992, Tokens per sec: 80741.35751978295, Loss: 2.2623767852783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9566 , TFLOPS: 101.19652691579009, Tokens per sec: 82690.2871732449, Loss: 2.253720998764038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9567 , TFLOPS: 98.73284451341236, Tokens per sec: 80677.14886143494, Loss: 2.2655115127563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9568 , TFLOPS: 98.96117272709635, Tokens per sec: 80863.72172252723, Loss: 2.268510580062866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9569 , TFLOPS: 100.08690921890692, Tokens per sec: 81783.58998902101, Loss: 2.271880626678467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9570 , TFLOPS: 99.48984667827165, Tokens per sec: 81295.71481731079, Loss: 2.2523088455200195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9571 , TFLOPS: 98.07986534661099, Tokens per sec: 80143.58277506239, Loss: 2.2475128173828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9572 , TFLOPS: 100.80379271309077, Tokens per sec: 82369.3739463415, Loss: 2.2567055225372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9573 , TFLOPS: 98.80953813303083, Tokens per sec: 80739.81719229459, Loss: 2.242295980453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9574 , TFLOPS: 101.19169968848722, Tokens per sec: 82686.34272155183, Loss: 2.2543632984161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9575 , TFLOPS: 99.36352048716687, Tokens per sec: 81192.49043463361, Loss: 2.2635676860809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9576 , TFLOPS: 101.27261680497648, Tokens per sec: 82752.46218042709, Loss: 2.245246648788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9577 , TFLOPS: 99.5206567363876, Tokens per sec: 81320.89050891879, Loss: 2.23549747467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9578 , TFLOPS: 100.01492538928476, Tokens per sec: 81724.7701488081, Loss: 2.258192539215088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9579 , TFLOPS: 99.95446044930367, Tokens per sec: 81675.36268984353, Loss: 2.2283692359924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9580 , TFLOPS: 99.11450703343621, Tokens per sec: 80989.01513141373, Loss: 2.2765705585479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9581 , TFLOPS: 99.50781019663435, Tokens per sec: 81310.39327058705, Loss: 2.2473952770233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9582 , TFLOPS: 100.15501378611364, Tokens per sec: 81839.23998405314, Loss: 2.2595527172088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9583 , TFLOPS: 101.25985093387672, Tokens per sec: 82742.03085852871, Loss: 2.2679858207702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9584 , TFLOPS: 98.24455209276606, Tokens per sec: 80278.15255475968, Loss: 2.2658276557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9585 , TFLOPS: 101.23538151817203, Tokens per sec: 82722.03626905763, Loss: 2.2444262504577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9586 , TFLOPS: 100.6132914564964, Tokens per sec: 82213.71046563963, Loss: 2.255401372909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9587 , TFLOPS: 98.62961378699706, Tokens per sec: 80592.79637748569, Loss: 2.2885243892669678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9588 , TFLOPS: 101.2515038291564, Tokens per sec: 82735.21022438815, Loss: 2.2669970989227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9589 , TFLOPS: 100.57991926011529, Tokens per sec: 82186.44118489987, Loss: 2.2694945335388184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9590 , TFLOPS: 98.82962236458505, Tokens per sec: 80756.22853491166, Loss: 2.2771432399749756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9591 , TFLOPS: 99.73826954522258, Tokens per sec: 81498.70753686992, Loss: 2.247241735458374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9592 , TFLOPS: 100.50596994051189, Tokens per sec: 82126.01529222708, Loss: 2.2729055881500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9593 , TFLOPS: 98.19408711902668, Tokens per sec: 80236.91632563325, Loss: 2.259214401245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9594 , TFLOPS: 100.67647058801546, Tokens per sec: 82265.3357603794, Loss: 2.2723984718322754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9595 , TFLOPS: 99.3324232402752, Tokens per sec: 81167.08007368408, Loss: 2.241279125213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9596 , TFLOPS: 99.2794683780434, Tokens per sec: 81123.80929257505, Loss: 2.246624708175659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9597 , TFLOPS: 99.41232935218521, Tokens per sec: 81232.37341469206, Loss: 2.251847505569458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9598 , TFLOPS: 100.57095467109752, Tokens per sec: 82179.11598844439, Loss: 2.263394355773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9599 , TFLOPS: 99.28760458083796, Tokens per sec: 81130.457593323, Loss: 2.269392967224121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9600 , TFLOPS: 100.00629704615001, Tokens per sec: 81717.71970751938, Loss: 2.270699977874756 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/optimizer.pt +[rank0]:[2024-08-28 22:25:32,403] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007906827000624617, 'preprocessing_with_comm': 0.002174867999201524, 'state_converting': 2.527937187000134, : 2.5396886129983614}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600, took 17.00s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007600 + eval ppl=7.675515174865723, eval loss=2.0380353927612305 +------------------------------------------------------------------ +iteration: 9601 , TFLOPS: 99.29151453278014, Tokens per sec: 81133.65251570605, Loss: 2.2577528953552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9602 , TFLOPS: 98.68689175173469, Tokens per sec: 80639.5996769388, Loss: 2.268043279647827 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9603 , TFLOPS: 98.36697433112698, Tokens per sec: 80378.18691715319, Loss: 2.2821860313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9604 , TFLOPS: 99.95800072175683, Tokens per sec: 81678.25553759973, Loss: 2.2543931007385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9605 , TFLOPS: 101.2507234209496, Tokens per sec: 82734.57253275288, Loss: 2.258899688720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9606 , TFLOPS: 98.54661467982544, Tokens per sec: 80524.97567042866, Loss: 2.2571041584014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9607 , TFLOPS: 100.61426094886083, Tokens per sec: 82214.50266280756, Loss: 2.2624247074127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9608 , TFLOPS: 99.26656329745792, Tokens per sec: 81113.26421902233, Loss: 2.2628936767578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9609 , TFLOPS: 100.62608495545688, Tokens per sec: 82224.16436297419, Loss: 2.2428529262542725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9610 , TFLOPS: 99.35882497649986, Tokens per sec: 81188.6536119944, Loss: 2.269928216934204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9611 , TFLOPS: 101.2437386752712, Tokens per sec: 82728.86511725571, Loss: 2.254035472869873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9612 , TFLOPS: 98.67831848739425, Tokens per sec: 80632.59424195076, Loss: 2.2700798511505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9613 , TFLOPS: 101.06788613977442, Tokens per sec: 82585.17148365431, Loss: 2.275270700454712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9614 , TFLOPS: 99.8871008349268, Tokens per sec: 81620.32141494546, Loss: 2.2634122371673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9615 , TFLOPS: 98.81415905256087, Tokens per sec: 80743.59306459564, Loss: 2.252164840698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9616 , TFLOPS: 100.675404192427, Tokens per sec: 82264.46438109259, Loss: 2.2626404762268066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9617 , TFLOPS: 99.45950604810935, Tokens per sec: 81270.92270736786, Loss: 2.2753443717956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9618 , TFLOPS: 101.25924428568864, Tokens per sec: 82741.53515067774, Loss: 2.2407538890838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9619 , TFLOPS: 100.07729677316992, Tokens per sec: 81775.73541216293, Loss: 2.296746015548706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9620 , TFLOPS: 101.33965934025234, Tokens per sec: 82807.24436182939, Loss: 2.277371406555176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9621 , TFLOPS: 99.82334335701506, Tokens per sec: 81568.22353848029, Loss: 2.271005630493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9622 , TFLOPS: 100.73054717970226, Tokens per sec: 82309.52313550196, Loss: 2.2639403343200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9623 , TFLOPS: 100.75426940881235, Tokens per sec: 82328.90717957234, Loss: 2.252603530883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9624 , TFLOPS: 100.62499670398336, Tokens per sec: 82223.2751246811, Loss: 2.2736833095550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9625 , TFLOPS: 98.73869827116751, Tokens per sec: 80681.93211758579, Loss: 2.2694144248962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9626 , TFLOPS: 100.67982163193334, Tokens per sec: 82268.07398462824, Loss: 2.2355685234069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9627 , TFLOPS: 100.75280923374035, Tokens per sec: 82327.71403293275, Loss: 2.244951009750366 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9628 , TFLOPS: 100.69948187777541, Tokens per sec: 82284.13887760554, Loss: 2.260176658630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9629 , TFLOPS: 100.12400737766764, Tokens per sec: 81813.90384953594, Loss: 2.2903242111206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9630 , TFLOPS: 99.55672008263004, Tokens per sec: 81350.35879748661, Loss: 2.274522066116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9631 , TFLOPS: 99.99933326047977, Tokens per sec: 81712.02941898444, Loss: 2.2617454528808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9632 , TFLOPS: 100.07511843877936, Tokens per sec: 81773.95543905708, Loss: 2.2475790977478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9633 , TFLOPS: 99.84785908770533, Tokens per sec: 81588.25597311836, Loss: 2.2514824867248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9634 , TFLOPS: 101.34865663591184, Tokens per sec: 82814.59628372376, Loss: 2.260342597961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9635 , TFLOPS: 98.68709464659136, Tokens per sec: 80639.76546754915, Loss: 2.261709213256836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9636 , TFLOPS: 99.74630837749221, Tokens per sec: 81505.27627365537, Loss: 2.2662200927734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9637 , TFLOPS: 101.26730356700723, Tokens per sec: 82748.12059690751, Loss: 2.2713778018951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9638 , TFLOPS: 98.28042424707786, Tokens per sec: 80307.46461547927, Loss: 2.273529529571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9639 , TFLOPS: 100.65507817624612, Tokens per sec: 82247.85547003298, Loss: 2.267721652984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9640 , TFLOPS: 97.73318972883841, Tokens per sec: 79860.30520355611, Loss: 2.257506847381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9641 , TFLOPS: 99.42856136168841, Tokens per sec: 81245.63700750623, Loss: 2.2633135318756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9642 , TFLOPS: 99.99279673163322, Tokens per sec: 81706.6882529987, Loss: 2.249232769012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9643 , TFLOPS: 101.19657728946981, Tokens per sec: 82690.32833487533, Loss: 2.26650333404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9644 , TFLOPS: 97.3778143277324, Tokens per sec: 79569.91881513571, Loss: 2.250822067260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9645 , TFLOPS: 100.66302073636118, Tokens per sec: 82254.34554036267, Loss: 2.258495569229126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9646 , TFLOPS: 98.70904668784388, Tokens per sec: 80657.70304555239, Loss: 2.259042739868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9647 , TFLOPS: 100.78135536354431, Tokens per sec: 82351.03981043832, Loss: 2.271001100540161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9648 , TFLOPS: 98.90326099063776, Tokens per sec: 80816.40055188621, Loss: 2.27947998046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9649 , TFLOPS: 100.58825652666259, Tokens per sec: 82193.25378001599, Loss: 2.264035701751709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9650 , TFLOPS: 99.38933197641596, Tokens per sec: 81213.5816669459, Loss: 2.2584550380706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9651 , TFLOPS: 101.14800500324795, Tokens per sec: 82650.63866944154, Loss: 2.2604269981384277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9652 , TFLOPS: 100.13631827223959, Tokens per sec: 81823.9633984013, Loss: 2.25277042388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9653 , TFLOPS: 99.4395926532245, Tokens per sec: 81254.65095979108, Loss: 2.260704278945923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9654 , TFLOPS: 100.81815107974275, Tokens per sec: 82381.10653735008, Loss: 2.254605293273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9655 , TFLOPS: 100.01465843033615, Tokens per sec: 81724.55200977907, Loss: 2.271030902862549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9656 , TFLOPS: 101.28710841029675, Tokens per sec: 82764.3036441814, Loss: 2.248952627182007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9657 , TFLOPS: 98.67565973321595, Tokens per sec: 80630.42170546926, Loss: 2.2516794204711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9658 , TFLOPS: 101.3404951166262, Tokens per sec: 82807.92729621926, Loss: 2.2639598846435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9659 , TFLOPS: 100.65038543350535, Tokens per sec: 82244.02090913756, Loss: 2.262984275817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9660 , TFLOPS: 100.58138702962954, Tokens per sec: 82187.6405371537, Loss: 2.2646753787994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9661 , TFLOPS: 100.58384837857292, Tokens per sec: 82189.6517687361, Loss: 2.2546632289886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9662 , TFLOPS: 100.00395399063287, Tokens per sec: 81715.80513654072, Loss: 2.2543046474456787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9663 , TFLOPS: 98.58873352061221, Tokens per sec: 80559.39205947095, Loss: 2.2674388885498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9664 , TFLOPS: 100.6256247788729, Tokens per sec: 82223.78834084143, Loss: 2.2639315128326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9665 , TFLOPS: 100.60197565058434, Tokens per sec: 82204.46402933412, Loss: 2.2679941654205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9666 , TFLOPS: 100.68368150285028, Tokens per sec: 82271.22798451637, Loss: 2.2805187702178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9667 , TFLOPS: 100.09728376987216, Tokens per sec: 81792.0673016794, Loss: 2.2696101665496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9668 , TFLOPS: 99.42534029667681, Tokens per sec: 81243.00499236766, Loss: 2.2595484256744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9669 , TFLOPS: 99.89397043494637, Tokens per sec: 81625.9347419607, Loss: 2.263643503189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9670 , TFLOPS: 99.53153953623033, Tokens per sec: 81329.78312481848, Loss: 2.257821798324585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9671 , TFLOPS: 100.09138520359087, Tokens per sec: 81787.24743132836, Loss: 2.2655413150787354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9672 , TFLOPS: 101.13780346857081, Tokens per sec: 82642.302732846, Loss: 2.2508959770202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9673 , TFLOPS: 98.75276994919541, Tokens per sec: 80693.43044793961, Loss: 2.248894691467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9674 , TFLOPS: 101.28936643438317, Tokens per sec: 82766.14873378912, Loss: 2.272214651107788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9675 , TFLOPS: 100.52761209909414, Tokens per sec: 82143.69965712338, Loss: 2.273700475692749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9676 , TFLOPS: 99.93348968361114, Tokens per sec: 81658.22693736064, Loss: 2.272716522216797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9677 , TFLOPS: 100.05793384529234, Tokens per sec: 81759.91346534806, Loss: 2.2741730213165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9678 , TFLOPS: 100.0754059050972, Tokens per sec: 81774.19033518543, Loss: 2.247124433517456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9679 , TFLOPS: 99.98810236911861, Tokens per sec: 81702.85236854438, Loss: 2.260624408721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9680 , TFLOPS: 99.932790724839, Tokens per sec: 81657.65580015512, Loss: 2.2724246978759766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9681 , TFLOPS: 99.87819682588976, Tokens per sec: 81613.04571995171, Loss: 2.2419238090515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9682 , TFLOPS: 98.19266695149658, Tokens per sec: 80235.75587018604, Loss: 2.2515792846679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9683 , TFLOPS: 100.6842949810612, Tokens per sec: 82271.72927335484, Loss: 2.2656519412994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9684 , TFLOPS: 98.8035740181013, Tokens per sec: 80734.94376045569, Loss: 2.2475855350494385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9685 , TFLOPS: 100.27267623826805, Tokens per sec: 81935.38500260949, Loss: 2.2621443271636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9686 , TFLOPS: 98.67444509651959, Tokens per sec: 80629.42919455712, Loss: 2.2438912391662598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9687 , TFLOPS: 100.0173948474124, Tokens per sec: 81726.78800661371, Loss: 2.2656497955322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9688 , TFLOPS: 99.46563772302721, Tokens per sec: 81275.93305678663, Loss: 2.268629312515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9689 , TFLOPS: 99.96291028462072, Tokens per sec: 81682.26726779921, Loss: 2.267298936843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9690 , TFLOPS: 100.07887136802769, Tokens per sec: 81777.02205415489, Loss: 2.264219284057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9691 , TFLOPS: 98.41265827279504, Tokens per sec: 80415.51644189942, Loss: 2.2680299282073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9692 , TFLOPS: 100.59920771385669, Tokens per sec: 82202.20227698093, Loss: 2.265387535095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9693 , TFLOPS: 100.11472071689697, Tokens per sec: 81806.31547996032, Loss: 2.2507572174072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9694 , TFLOPS: 100.70360111445387, Tokens per sec: 82287.5048119341, Loss: 2.288278102874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9695 , TFLOPS: 99.38935493664354, Tokens per sec: 81213.6004283389, Loss: 2.267688274383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9696 , TFLOPS: 100.7270152451998, Tokens per sec: 82306.63710089993, Loss: 2.2711501121520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9697 , TFLOPS: 100.62184583454888, Tokens per sec: 82220.7004681556, Loss: 2.2474303245544434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9698 , TFLOPS: 99.39439736466824, Tokens per sec: 81217.72072608152, Loss: 2.2600972652435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9699 , TFLOPS: 100.77148323101757, Tokens per sec: 82342.97303681941, Loss: 2.2426326274871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9700 , TFLOPS: 99.43917676030136, Tokens per sec: 81254.31112297754, Loss: 2.2545511722564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9701 , TFLOPS: 99.167717204802, Tokens per sec: 81032.49453218839, Loss: 2.236219644546509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9702 , TFLOPS: 100.15090592851962, Tokens per sec: 81835.8833478674, Loss: 2.274501323699951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9703 , TFLOPS: 100.09691217262281, Tokens per sec: 81791.76366000119, Loss: 2.271974802017212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9704 , TFLOPS: 99.78015387252329, Tokens per sec: 81532.93229890581, Loss: 2.2517356872558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9705 , TFLOPS: 99.31577045214492, Tokens per sec: 81153.47265182192, Loss: 2.272167682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9706 , TFLOPS: 100.11444711365834, Tokens per sec: 81806.09191171087, Loss: 2.275503396987915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9707 , TFLOPS: 99.88124331708885, Tokens per sec: 81615.53508633414, Loss: 2.262375593185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9708 , TFLOPS: 99.64330152710536, Tokens per sec: 81421.10672457206, Loss: 2.24119234085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9709 , TFLOPS: 100.06743383818129, Tokens per sec: 81767.67615408919, Loss: 2.2470784187316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9710 , TFLOPS: 101.29866528358166, Tokens per sec: 82773.74706284286, Loss: 2.2633731365203857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9711 , TFLOPS: 98.7567096080144, Tokens per sec: 80696.64964457646, Loss: 2.255631446838379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9712 , TFLOPS: 101.31718497997733, Tokens per sec: 82788.87998350718, Loss: 2.2367124557495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9713 , TFLOPS: 99.9112524407827, Tokens per sec: 81640.05631380796, Loss: 2.2665460109710693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9714 , TFLOPS: 99.85536324902905, Tokens per sec: 81594.38781651028, Loss: 2.249237537384033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9715 , TFLOPS: 100.69031806880453, Tokens per sec: 82276.650893398, Loss: 2.2778308391571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9716 , TFLOPS: 99.4363286844377, Tokens per sec: 81251.98388687335, Loss: 2.2590744495391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9717 , TFLOPS: 99.8307039644096, Tokens per sec: 81574.23808026125, Loss: 2.2633419036865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9718 , TFLOPS: 100.65126391357327, Tokens per sec: 82244.73873781515, Loss: 2.2544403076171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9719 , TFLOPS: 99.28197474779365, Tokens per sec: 81125.85731181769, Loss: 2.2517566680908203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9720 , TFLOPS: 99.43719328617203, Tokens per sec: 81252.6903752073, Loss: 2.2601163387298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9721 , TFLOPS: 99.85490269850268, Tokens per sec: 81594.01148881955, Loss: 2.2684412002563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9722 , TFLOPS: 99.26684427783397, Tokens per sec: 81113.49381532067, Loss: 2.284646511077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9723 , TFLOPS: 98.91513752508108, Tokens per sec: 80826.10517390903, Loss: 2.2763314247131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9724 , TFLOPS: 99.44175073095371, Tokens per sec: 81256.41438065749, Loss: 2.2417683601379395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9725 , TFLOPS: 100.45383917830631, Tokens per sec: 82083.4179044637, Loss: 2.2554991245269775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9726 , TFLOPS: 99.3106917599811, Tokens per sec: 81149.3227217177, Loss: 2.2582621574401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9727 , TFLOPS: 99.58312458751735, Tokens per sec: 81371.93459814246, Loss: 2.2679052352905273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9728 , TFLOPS: 100.06240373410732, Tokens per sec: 81763.56592656394, Loss: 2.2663989067077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9729 , TFLOPS: 98.83278483365267, Tokens per sec: 80758.812669796, Loss: 2.263232469558716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9730 , TFLOPS: 99.94092169715303, Tokens per sec: 81664.29982694276, Loss: 2.2546777725219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9731 , TFLOPS: 99.62304291959093, Tokens per sec: 81404.55289486908, Loss: 2.2614805698394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9732 , TFLOPS: 100.00931452353818, Tokens per sec: 81720.18536597004, Loss: 2.2785027027130127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9733 , TFLOPS: 99.92260279267637, Tokens per sec: 81649.33097852433, Loss: 2.248725414276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9734 , TFLOPS: 100.20534242921575, Tokens per sec: 81880.3648139064, Loss: 2.2695488929748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9735 , TFLOPS: 100.71933355627448, Tokens per sec: 82300.3601951348, Loss: 2.2892141342163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9736 , TFLOPS: 100.05629261364847, Tokens per sec: 81758.57237272279, Loss: 2.2725412845611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9737 , TFLOPS: 100.79902323745101, Tokens per sec: 82365.47668501902, Loss: 2.2610762119293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9738 , TFLOPS: 98.6451712530639, Tokens per sec: 80605.50878349371, Loss: 2.2543392181396484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9739 , TFLOPS: 99.3131491325591, Tokens per sec: 81151.33070410958, Loss: 2.2541286945343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9740 , TFLOPS: 100.18815075273206, Tokens per sec: 81866.31705249846, Loss: 2.249626398086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9741 , TFLOPS: 99.49107186762012, Tokens per sec: 81296.7159510666, Loss: 2.273540735244751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9742 , TFLOPS: 101.08601983094083, Tokens per sec: 82599.98898950912, Loss: 2.2775063514709473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9743 , TFLOPS: 98.35547700116861, Tokens per sec: 80368.79215287659, Loss: 2.2648768424987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9744 , TFLOPS: 99.98226751618631, Tokens per sec: 81698.08456001093, Loss: 2.268061399459839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9745 , TFLOPS: 100.40313263542929, Tokens per sec: 82041.98428297635, Loss: 2.2641139030456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9746 , TFLOPS: 99.47476601763778, Tokens per sec: 81283.39202129611, Loss: 2.2646660804748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9747 , TFLOPS: 100.59746283417928, Tokens per sec: 82200.77649087932, Loss: 2.2462592124938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9748 , TFLOPS: 101.3707807234348, Tokens per sec: 82832.67444516344, Loss: 2.2593607902526855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9749 , TFLOPS: 98.83698676858977, Tokens per sec: 80762.24617900053, Loss: 2.2683286666870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9750 , TFLOPS: 101.37542399602344, Tokens per sec: 82836.4685827241, Loss: 2.2511651515960693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9751 , TFLOPS: 99.2508342475355, Tokens per sec: 81100.41160742896, Loss: 2.263000726699829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9752 , TFLOPS: 99.88466048664893, Tokens per sec: 81618.32734354745, Loss: 2.2627131938934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9753 , TFLOPS: 99.90727796254698, Tokens per sec: 81636.80866532939, Loss: 2.2540016174316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9754 , TFLOPS: 99.51812218372322, Tokens per sec: 81318.81946069153, Loss: 2.2546215057373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9755 , TFLOPS: 99.46552373769653, Tokens per sec: 81275.83991643866, Loss: 2.246833562850952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9756 , TFLOPS: 99.29744813130866, Tokens per sec: 81138.50101180813, Loss: 2.2446465492248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9757 , TFLOPS: 101.25782402920456, Tokens per sec: 82740.37462254392, Loss: 2.2679672241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9758 , TFLOPS: 97.90398429381732, Tokens per sec: 79999.8658392436, Loss: 2.2571778297424316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9759 , TFLOPS: 99.4128684500828, Tokens per sec: 81232.8139254618, Loss: 2.2396552562713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9760 , TFLOPS: 99.38952641277717, Tokens per sec: 81213.74054590188, Loss: 2.261168956756592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9761 , TFLOPS: 98.8853047414781, Tokens per sec: 80801.72803846287, Loss: 2.2838058471679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9762 , TFLOPS: 98.85442034470395, Tokens per sec: 80776.4916028236, Loss: 2.267590045928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9763 , TFLOPS: 100.68069844155644, Tokens per sec: 82268.79044834235, Loss: 2.2644009590148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9764 , TFLOPS: 98.88911224547886, Tokens per sec: 80804.83924799575, Loss: 2.2576076984405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9765 , TFLOPS: 100.67388609542556, Tokens per sec: 82263.22390495341, Loss: 2.2665212154388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9766 , TFLOPS: 99.5649473949959, Tokens per sec: 81357.08154621061, Loss: 2.263005018234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9767 , TFLOPS: 100.76001648215437, Tokens per sec: 82333.60326114293, Loss: 2.267653465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9768 , TFLOPS: 99.43966098777933, Tokens per sec: 81254.70679771491, Loss: 2.256300449371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9769 , TFLOPS: 100.7054037516319, Tokens per sec: 82288.97779317627, Loss: 2.258312463760376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9770 , TFLOPS: 98.80310285424235, Tokens per sec: 80734.55876033772, Loss: 2.244018316268921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9771 , TFLOPS: 99.80883108259034, Tokens per sec: 81556.36518546868, Loss: 2.2596025466918945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9772 , TFLOPS: 99.87666440543094, Tokens per sec: 81611.7935397468, Loss: 2.277147054672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9773 , TFLOPS: 99.73341444723367, Tokens per sec: 81494.74031132183, Loss: 2.265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9774 , TFLOPS: 100.03657181694507, Tokens per sec: 81742.45800207797, Loss: 2.251235246658325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9775 , TFLOPS: 99.03224620928172, Tokens per sec: 80921.7976944151, Loss: 2.270084857940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9776 , TFLOPS: 100.70409773470915, Tokens per sec: 82287.91061312883, Loss: 2.2700328826904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9777 , TFLOPS: 100.03287697743225, Tokens per sec: 81739.43885359842, Loss: 2.262977361679077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9778 , TFLOPS: 100.1575979466175, Tokens per sec: 81841.35156812305, Loss: 2.2477073669433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9779 , TFLOPS: 100.76224281384576, Tokens per sec: 82335.42245407884, Loss: 2.270279884338379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9780 , TFLOPS: 100.53039525728965, Tokens per sec: 82145.97384732979, Loss: 2.24664306640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9781 , TFLOPS: 98.22998545171, Tokens per sec: 80266.24977737409, Loss: 2.2786498069763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9782 , TFLOPS: 100.7115765394794, Tokens per sec: 82294.02173702815, Loss: 2.252560615539551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9783 , TFLOPS: 98.78320761963838, Tokens per sec: 80718.3018519939, Loss: 2.2747044563293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9784 , TFLOPS: 99.65508744273505, Tokens per sec: 81430.73729962955, Loss: 2.264904260635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9785 , TFLOPS: 100.66036994697829, Tokens per sec: 82252.17951212055, Loss: 2.2683775424957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9786 , TFLOPS: 100.11893809130098, Tokens per sec: 81809.7616051507, Loss: 2.2609176635742188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9787 , TFLOPS: 99.96291777767199, Tokens per sec: 81682.2733905643, Loss: 2.2567055225372314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9788 , TFLOPS: 99.96279907126721, Tokens per sec: 81682.17639250516, Loss: 2.264542579650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9789 , TFLOPS: 98.11710013503279, Tokens per sec: 80174.00827919057, Loss: 2.255910873413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9790 , TFLOPS: 99.99775334530972, Tokens per sec: 81710.73842962836, Loss: 2.2668492794036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9791 , TFLOPS: 100.09873570829706, Tokens per sec: 81793.25371794259, Loss: 2.2556498050689697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9792 , TFLOPS: 98.6824430870629, Tokens per sec: 80635.96455851664, Loss: 2.2756035327911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9793 , TFLOPS: 99.01427620462198, Tokens per sec: 80907.11394101856, Loss: 2.2541868686676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9794 , TFLOPS: 98.91714253698075, Tokens per sec: 80827.74352074585, Loss: 2.2589495182037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9795 , TFLOPS: 100.57519398424544, Tokens per sec: 82182.58004034718, Loss: 2.251647710800171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9796 , TFLOPS: 99.564070914227, Tokens per sec: 81356.36535121173, Loss: 2.2508339881896973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9797 , TFLOPS: 99.09998857999877, Tokens per sec: 80977.15172937176, Loss: 2.253941297531128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9798 , TFLOPS: 100.66668951565647, Tokens per sec: 82257.34339436765, Loss: 2.2547881603240967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9799 , TFLOPS: 98.2970671268412, Tokens per sec: 80321.06394095974, Loss: 2.257490634918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9800 , TFLOPS: 100.35144785114781, Tokens per sec: 81999.75131525526, Loss: 2.2596282958984375 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/optimizer.pt +[rank0]:[2024-08-28 23:19:29,960] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007724372000666335, 'preprocessing_with_comm': 0.0016297959991788957, 'state_converting': 2.4405956530008552, : 2.4515937910000503}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800, took 16.23s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0007800 + eval ppl=7.929728984832764, eval loss=2.0706188678741455 +------------------------------------------------------------------ +iteration: 9801 , TFLOPS: 99.20996225319169, Tokens per sec: 81067.01404871189, Loss: 2.2656521797180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9802 , TFLOPS: 99.69681501425681, Tokens per sec: 81464.83397248322, Loss: 2.246473550796509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9803 , TFLOPS: 98.51650424942491, Tokens per sec: 80500.37166263694, Loss: 2.272878408432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9804 , TFLOPS: 98.70338194451865, Tokens per sec: 80653.07423795789, Loss: 2.271331548690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9805 , TFLOPS: 100.03175948373729, Tokens per sec: 81738.5257207334, Loss: 2.2470593452453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9806 , TFLOPS: 98.99416641336072, Tokens per sec: 80890.6817129069, Loss: 2.2759757041931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9807 , TFLOPS: 100.21751462065158, Tokens per sec: 81890.3110248687, Loss: 2.2606236934661865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9808 , TFLOPS: 100.63287939882443, Tokens per sec: 82229.71627755434, Loss: 2.2709336280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9809 , TFLOPS: 100.11082510489373, Tokens per sec: 81803.1322751105, Loss: 2.258833169937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9810 , TFLOPS: 101.23015420486112, Tokens per sec: 82717.76489678826, Loss: 2.2587461471557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9811 , TFLOPS: 100.02851968963294, Tokens per sec: 81735.87840157103, Loss: 2.2620158195495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9812 , TFLOPS: 100.55886936716091, Tokens per sec: 82169.24077549439, Loss: 2.2669668197631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9813 , TFLOPS: 100.30175616729004, Tokens per sec: 81959.14696119732, Loss: 2.2579171657562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9814 , TFLOPS: 100.59176709688403, Tokens per sec: 82196.12235731405, Loss: 2.2497267723083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9815 , TFLOPS: 99.34287420495629, Tokens per sec: 81175.61982595676, Loss: 2.275718927383423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9816 , TFLOPS: 101.2755227128375, Tokens per sec: 82754.83667254506, Loss: 2.2488436698913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9817 , TFLOPS: 99.98458664949067, Tokens per sec: 81699.97958353371, Loss: 2.264920234680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9818 , TFLOPS: 99.15435516973142, Tokens per sec: 81021.57606936283, Loss: 2.2738194465637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9819 , TFLOPS: 100.08880440247277, Tokens per sec: 81785.13859229903, Loss: 2.2853522300720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9820 , TFLOPS: 100.61898905153379, Tokens per sec: 82218.36611721382, Loss: 2.274174690246582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9821 , TFLOPS: 100.62128604101369, Tokens per sec: 82220.24304644762, Loss: 2.224578619003296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9822 , TFLOPS: 99.24977478417883, Tokens per sec: 81099.5458926472, Loss: 2.2498223781585693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9823 , TFLOPS: 99.42059708609374, Tokens per sec: 81239.12919289914, Loss: 2.2754385471343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9824 , TFLOPS: 98.62257236590027, Tokens per sec: 80587.0426510452, Loss: 2.293567180633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9825 , TFLOPS: 101.23833143436298, Tokens per sec: 82724.44672151485, Loss: 2.2645437717437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9826 , TFLOPS: 99.45043028205204, Tokens per sec: 81263.50666529144, Loss: 2.2884480953216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9827 , TFLOPS: 98.77509203593759, Tokens per sec: 80711.67039963827, Loss: 2.27139949798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9828 , TFLOPS: 99.94576421475918, Tokens per sec: 81668.25677273625, Loss: 2.254915475845337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9829 , TFLOPS: 98.84713386344636, Tokens per sec: 80770.53763141733, Loss: 2.261737585067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9830 , TFLOPS: 100.12165592514373, Tokens per sec: 81811.98241714688, Loss: 2.2446675300598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9831 , TFLOPS: 98.69468660687703, Tokens per sec: 80645.969053733, Loss: 2.245997905731201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9832 , TFLOPS: 100.07114527038134, Tokens per sec: 81770.70886088061, Loss: 2.2764878273010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9833 , TFLOPS: 100.00450920356599, Tokens per sec: 81716.25881532082, Loss: 2.2670938968658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9834 , TFLOPS: 99.12301337492013, Tokens per sec: 80995.96588201296, Loss: 2.2494499683380127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9835 , TFLOPS: 99.62311441788444, Tokens per sec: 81404.61131796522, Loss: 2.253119707107544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9836 , TFLOPS: 98.73726963546198, Tokens per sec: 80680.76474257445, Loss: 2.2543890476226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9837 , TFLOPS: 100.63679286904019, Tokens per sec: 82232.91407480933, Loss: 2.2699594497680664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9838 , TFLOPS: 99.60993125366102, Tokens per sec: 81393.83901511342, Loss: 2.2582147121429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9839 , TFLOPS: 100.02152472673453, Tokens per sec: 81730.16263732038, Loss: 2.2485575675964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9840 , TFLOPS: 100.08114648303044, Tokens per sec: 81778.8811091903, Loss: 2.2624614238739014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9841 , TFLOPS: 100.70984975479138, Tokens per sec: 82292.61073680817, Loss: 2.262092351913452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9842 , TFLOPS: 99.57982489894864, Tokens per sec: 81369.23833767143, Loss: 2.2736403942108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9843 , TFLOPS: 100.03658809663571, Tokens per sec: 81742.47130463226, Loss: 2.270277976989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9844 , TFLOPS: 100.21892683062264, Tokens per sec: 81891.46497798958, Loss: 2.253641366958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9845 , TFLOPS: 99.4330414260656, Tokens per sec: 81249.29778343595, Loss: 2.2798826694488525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9846 , TFLOPS: 100.02067318655887, Tokens per sec: 81729.46682192224, Loss: 2.2603490352630615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9847 , TFLOPS: 100.22015835735931, Tokens per sec: 81892.47129018842, Loss: 2.2543084621429443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9848 , TFLOPS: 101.29293035074231, Tokens per sec: 82769.06090158955, Loss: 2.2625253200531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9849 , TFLOPS: 99.40945126472003, Tokens per sec: 81230.0216553357, Loss: 2.2827324867248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9850 , TFLOPS: 100.65424742669782, Tokens per sec: 82247.17664319168, Loss: 2.27707839012146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9851 , TFLOPS: 100.55429729270932, Tokens per sec: 82165.50481576432, Loss: 2.27292537689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9852 , TFLOPS: 100.54824027648104, Tokens per sec: 82160.55547188272, Loss: 2.266188859939575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9853 , TFLOPS: 98.79182830917954, Tokens per sec: 80725.34603933428, Loss: 2.260044574737549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9854 , TFLOPS: 101.18468398908865, Tokens per sec: 82680.61001297309, Loss: 2.2567334175109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9855 , TFLOPS: 100.0119731832666, Tokens per sec: 81722.35782527413, Loss: 2.2547008991241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9856 , TFLOPS: 98.66916231623708, Tokens per sec: 80625.11249879742, Loss: 2.2527451515197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9857 , TFLOPS: 100.19397511124687, Tokens per sec: 81871.07628577322, Loss: 2.269670248031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9858 , TFLOPS: 101.28654970718387, Tokens per sec: 82763.84711348556, Loss: 2.241941452026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9859 , TFLOPS: 99.29067607714069, Tokens per sec: 81132.96739201929, Loss: 2.2820022106170654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9860 , TFLOPS: 99.93761587110646, Tokens per sec: 81661.59855138058, Loss: 2.2720556259155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9861 , TFLOPS: 98.74050863284461, Tokens per sec: 80683.41141071501, Loss: 2.264357566833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9862 , TFLOPS: 98.84224567254562, Tokens per sec: 80766.54336479909, Loss: 2.2570841312408447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9863 , TFLOPS: 100.68441522411572, Tokens per sec: 82271.82752705005, Loss: 2.283240556716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9864 , TFLOPS: 99.18867327928149, Tokens per sec: 81049.61828010292, Loss: 2.2353248596191406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9865 , TFLOPS: 98.30777751375119, Tokens per sec: 80329.81567381378, Loss: 2.2522213459014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9866 , TFLOPS: 99.91971202825243, Tokens per sec: 81646.96886049863, Loss: 2.257279872894287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9867 , TFLOPS: 98.76779761996413, Tokens per sec: 80705.7099445714, Loss: 2.2451727390289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9868 , TFLOPS: 98.86299139662302, Tokens per sec: 80783.49522998519, Loss: 2.236196756362915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9869 , TFLOPS: 97.97984313041034, Tokens per sec: 80061.85204740394, Loss: 2.274869441986084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9870 , TFLOPS: 99.55034409150369, Tokens per sec: 81345.14881100465, Loss: 2.26829195022583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9871 , TFLOPS: 99.65912986617964, Tokens per sec: 81434.04046788735, Loss: 2.2774696350097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9872 , TFLOPS: 99.00713197021587, Tokens per sec: 80901.27620317637, Loss: 2.272972822189331 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9873 , TFLOPS: 99.49216208590074, Tokens per sec: 81297.60679648841, Loss: 2.2605671882629395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9874 , TFLOPS: 98.97542194084748, Tokens per sec: 80875.36512189069, Loss: 2.2601664066314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9875 , TFLOPS: 100.6160091792477, Tokens per sec: 82215.93118686012, Loss: 2.2823047637939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9876 , TFLOPS: 99.41755832847471, Tokens per sec: 81236.64614582404, Loss: 2.2768163681030273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9877 , TFLOPS: 100.76109498424769, Tokens per sec: 82334.4845329665, Loss: 2.2576723098754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9878 , TFLOPS: 99.52013510988445, Tokens per sec: 81320.4642744752, Loss: 2.265352725982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9879 , TFLOPS: 100.71839429712466, Tokens per sec: 82299.5927023048, Loss: 2.23956036567688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9880 , TFLOPS: 98.59775160019582, Tokens per sec: 80566.76096444474, Loss: 2.2427189350128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9881 , TFLOPS: 100.0149208102763, Tokens per sec: 81724.76640718243, Loss: 2.2583017349243164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9882 , TFLOPS: 99.71515065241228, Tokens per sec: 81479.81649442136, Loss: 2.2664761543273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9883 , TFLOPS: 100.05147791535211, Tokens per sec: 81754.63815880349, Loss: 2.270634412765503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9884 , TFLOPS: 100.64301546902203, Tokens per sec: 82237.99872143849, Loss: 2.250844955444336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9885 , TFLOPS: 99.54302794515147, Tokens per sec: 81339.1705994859, Loss: 2.2649292945861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9886 , TFLOPS: 98.36886592738132, Tokens per sec: 80379.73258914663, Loss: 2.2544963359832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9887 , TFLOPS: 99.99939243905261, Tokens per sec: 81712.07777531972, Loss: 2.260969400405884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9888 , TFLOPS: 100.09504912966857, Tokens per sec: 81790.2413196443, Loss: 2.267094135284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9889 , TFLOPS: 100.66111447955763, Tokens per sec: 82252.7878888571, Loss: 2.270228385925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9890 , TFLOPS: 101.2082884437927, Tokens per sec: 82699.89782054449, Loss: 2.2554755210876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9891 , TFLOPS: 98.6599472756903, Tokens per sec: 80617.5826519504, Loss: 2.2803614139556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9892 , TFLOPS: 100.67472095031357, Tokens per sec: 82263.90608637349, Loss: 2.274980068206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9893 , TFLOPS: 99.44693881240963, Tokens per sec: 81260.6536955682, Loss: 2.2750420570373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9894 , TFLOPS: 99.95166784241175, Tokens per sec: 81673.08077886405, Loss: 2.2721590995788574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9895 , TFLOPS: 98.3270508052344, Tokens per sec: 80345.56437642338, Loss: 2.2553648948669434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9896 , TFLOPS: 101.23535507370356, Tokens per sec: 82722.0146606017, Loss: 2.277468204498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9897 , TFLOPS: 99.87484411847211, Tokens per sec: 81610.30613641438, Loss: 2.2546639442443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9898 , TFLOPS: 99.19750221414431, Tokens per sec: 81056.83263005633, Loss: 2.253387212753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9899 , TFLOPS: 98.29143535673455, Tokens per sec: 80316.46207663094, Loss: 2.259124994277954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9900 , TFLOPS: 99.10336710779376, Tokens per sec: 80979.91241140409, Loss: 2.2699882984161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9901 , TFLOPS: 99.89014844104868, Tokens per sec: 81622.81169236, Loss: 2.2714550495147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9902 , TFLOPS: 99.40362868305665, Tokens per sec: 81225.26387397197, Loss: 2.2641489505767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9903 , TFLOPS: 98.32822924721154, Tokens per sec: 80346.5273116987, Loss: 2.2501344680786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9904 , TFLOPS: 100.6463960954687, Tokens per sec: 82240.76111833312, Loss: 2.266111373901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9905 , TFLOPS: 98.7754943168074, Tokens per sec: 80711.99911369264, Loss: 2.2520334720611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9906 , TFLOPS: 99.45162075028254, Tokens per sec: 81264.47942752797, Loss: 2.2665700912475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9907 , TFLOPS: 98.74908943383552, Tokens per sec: 80690.42300409416, Loss: 2.259476900100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9908 , TFLOPS: 99.74294223111282, Tokens per sec: 81502.52570879646, Loss: 2.2548160552978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9909 , TFLOPS: 100.09836404411334, Tokens per sec: 81792.95002157059, Loss: 2.261481761932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9910 , TFLOPS: 98.33452817814918, Tokens per sec: 80351.67433031673, Loss: 2.258622407913208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9911 , TFLOPS: 99.59174868514835, Tokens per sec: 81378.98157032077, Loss: 2.262937307357788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9912 , TFLOPS: 98.72741158706461, Tokens per sec: 80672.70947746023, Loss: 2.245353937149048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9913 , TFLOPS: 100.56879713004558, Tokens per sec: 82177.3530061108, Loss: 2.2909159660339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9914 , TFLOPS: 99.54597634698726, Tokens per sec: 81341.57981452454, Loss: 2.223123550415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9915 , TFLOPS: 100.68535927945946, Tokens per sec: 82272.59893897355, Loss: 2.256850004196167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9916 , TFLOPS: 99.35453669946281, Tokens per sec: 81185.14955043736, Loss: 2.2698776721954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9917 , TFLOPS: 99.79476003235487, Tokens per sec: 81544.8673681002, Loss: 2.255497694015503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9918 , TFLOPS: 99.56697613886148, Tokens per sec: 81358.7392850477, Loss: 2.2295753955841064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9919 , TFLOPS: 99.96217234170093, Tokens per sec: 81681.66427564311, Loss: 2.2602829933166504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9920 , TFLOPS: 99.25053554150975, Tokens per sec: 81100.16752704594, Loss: 2.2617034912109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9921 , TFLOPS: 100.19535022484455, Tokens per sec: 81872.19992649247, Loss: 2.2638938426971436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9922 , TFLOPS: 99.85992627099246, Tokens per sec: 81598.1163792192, Loss: 2.2591052055358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9923 , TFLOPS: 99.38442181820085, Tokens per sec: 81209.56945026961, Loss: 2.2598772048950195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9924 , TFLOPS: 101.34123773847652, Tokens per sec: 82808.53411164995, Loss: 2.265564203262329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9925 , TFLOPS: 99.4170182070839, Tokens per sec: 81236.20479873165, Loss: 2.251887083053589 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9926 , TFLOPS: 100.63859445877925, Tokens per sec: 82234.38620016217, Loss: 2.2502946853637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9927 , TFLOPS: 100.016876492519, Tokens per sec: 81726.36444548667, Loss: 2.253980875015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9928 , TFLOPS: 100.52793727265943, Tokens per sec: 82143.96536481427, Loss: 2.2577691078186035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9929 , TFLOPS: 99.36006313646928, Tokens per sec: 81189.66534437849, Loss: 2.2471401691436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9930 , TFLOPS: 100.62863396640805, Tokens per sec: 82226.24722543977, Loss: 2.2725160121917725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9931 , TFLOPS: 99.40740045820166, Tokens per sec: 81228.3458885371, Loss: 2.2665517330169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9932 , TFLOPS: 98.72930999267959, Tokens per sec: 80674.26071355752, Loss: 2.254049777984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9933 , TFLOPS: 100.00531438193832, Tokens per sec: 81716.916747296, Loss: 2.262824058532715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9934 , TFLOPS: 100.66403983943908, Tokens per sec: 82255.17827572169, Loss: 2.289393424987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9935 , TFLOPS: 100.65228060180819, Tokens per sec: 82245.56949994378, Loss: 2.2506847381591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9936 , TFLOPS: 99.43415799065757, Tokens per sec: 81250.21015710698, Loss: 2.262971878051758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9937 , TFLOPS: 98.77004346873404, Tokens per sec: 80707.54508541457, Loss: 2.253641366958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9938 , TFLOPS: 99.96185773830555, Tokens per sec: 81681.40720511014, Loss: 2.272435426712036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9939 , TFLOPS: 99.95918805171594, Tokens per sec: 81679.22573547391, Loss: 2.2542147636413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9940 , TFLOPS: 100.10128169289288, Tokens per sec: 81795.33410749532, Loss: 2.2533326148986816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9941 , TFLOPS: 98.95191049850085, Tokens per sec: 80856.15331711089, Loss: 2.277160882949829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9942 , TFLOPS: 99.35542666035613, Tokens per sec: 81185.87676039292, Loss: 2.2505910396575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9943 , TFLOPS: 99.57862869156376, Tokens per sec: 81368.26088582411, Loss: 2.269192934036255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9944 , TFLOPS: 99.5101082718221, Tokens per sec: 81312.27108698069, Loss: 2.2518398761749268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9945 , TFLOPS: 98.7127473409029, Tokens per sec: 80660.72694443021, Loss: 2.2570254802703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9946 , TFLOPS: 99.60546121223017, Tokens per sec: 81390.18642919116, Loss: 2.2687346935272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9947 , TFLOPS: 100.0763050471893, Tokens per sec: 81774.92504733487, Loss: 2.247547149658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9948 , TFLOPS: 98.34232482114213, Tokens per sec: 80358.04516800972, Loss: 2.251188278198242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9949 , TFLOPS: 99.4202247093154, Tokens per sec: 81238.82491424771, Loss: 2.2776005268096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9950 , TFLOPS: 99.30751241609451, Tokens per sec: 81146.72479798438, Loss: 2.2555508613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9951 , TFLOPS: 99.86922626050175, Tokens per sec: 81605.71564005027, Loss: 2.266695499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9952 , TFLOPS: 98.92345301341763, Tokens per sec: 80832.89997348854, Loss: 2.273097276687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9953 , TFLOPS: 100.75057083307632, Tokens per sec: 82325.88497812861, Loss: 2.2595784664154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9954 , TFLOPS: 98.91700256547158, Tokens per sec: 80827.62914642252, Loss: 2.272367238998413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9955 , TFLOPS: 100.15653529874828, Tokens per sec: 81840.48325119412, Loss: 2.26816725730896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9956 , TFLOPS: 99.38312703556474, Tokens per sec: 81208.51145004698, Loss: 2.2603676319122314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9957 , TFLOPS: 100.03901521833563, Tokens per sec: 81744.45456825294, Loss: 2.2565886974334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9958 , TFLOPS: 98.95854892241589, Tokens per sec: 80861.57774418009, Loss: 2.260040521621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9959 , TFLOPS: 100.2793165057087, Tokens per sec: 81940.81093607093, Loss: 2.2547733783721924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9960 , TFLOPS: 101.32496671829617, Tokens per sec: 82795.23864220726, Loss: 2.2603776454925537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9961 , TFLOPS: 99.61586509183344, Tokens per sec: 81398.6877070347, Loss: 2.247208833694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9962 , TFLOPS: 100.66425499726559, Tokens per sec: 82255.35408672038, Loss: 2.266909122467041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9963 , TFLOPS: 98.70527697471476, Tokens per sec: 80654.62271591356, Loss: 2.2646126747131348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9964 , TFLOPS: 101.31062774522489, Tokens per sec: 82783.52189819272, Loss: 2.2452683448791504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9965 , TFLOPS: 99.19105343100243, Tokens per sec: 81051.56316334466, Loss: 2.286057710647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9966 , TFLOPS: 101.26951496914995, Tokens per sec: 82749.92759052488, Loss: 2.266716241836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9967 , TFLOPS: 98.83746994936538, Tokens per sec: 80762.64099845047, Loss: 2.250568389892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9968 , TFLOPS: 101.26241902457245, Tokens per sec: 82744.12931154473, Loss: 2.232666015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9969 , TFLOPS: 98.20796987139597, Tokens per sec: 80248.26027996805, Loss: 2.2546987533569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9970 , TFLOPS: 100.04161563663831, Tokens per sec: 81746.57943698879, Loss: 2.250352621078491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9971 , TFLOPS: 99.37683627347394, Tokens per sec: 81203.37110640395, Loss: 2.26938533782959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9972 , TFLOPS: 100.67352435683269, Tokens per sec: 82262.9283190372, Loss: 2.2753822803497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9973 , TFLOPS: 100.55839870834039, Tokens per sec: 82168.85618805629, Loss: 2.2582926750183105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9974 , TFLOPS: 99.34346139670086, Tokens per sec: 81176.09963544691, Loss: 2.2693967819213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9975 , TFLOPS: 98.94466073445479, Tokens per sec: 80850.2293482833, Loss: 2.272082805633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9976 , TFLOPS: 99.54032338410792, Tokens per sec: 81336.9606330356, Loss: 2.264618158340454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9977 , TFLOPS: 101.21557191683893, Tokens per sec: 82705.84933386384, Loss: 2.2531278133392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9978 , TFLOPS: 98.95604429182751, Tokens per sec: 80859.5311460514, Loss: 2.2667484283447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9979 , TFLOPS: 99.34438879538843, Tokens per sec: 81176.85743678789, Loss: 2.2620491981506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9980 , TFLOPS: 99.99378624898345, Tokens per sec: 81707.49681309803, Loss: 2.2643017768859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9981 , TFLOPS: 99.93545872163239, Tokens per sec: 81659.83588901535, Loss: 2.2545671463012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9982 , TFLOPS: 100.17076669969116, Tokens per sec: 81852.1120952534, Loss: 2.274021863937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9983 , TFLOPS: 99.36663967308282, Tokens per sec: 81195.03920174052, Loss: 2.2608814239501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9984 , TFLOPS: 99.46596557017557, Tokens per sec: 81276.20094913107, Loss: 2.2543833255767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9985 , TFLOPS: 100.86294621729317, Tokens per sec: 82417.70979737195, Loss: 2.2882771492004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9986 , TFLOPS: 97.10903063821759, Tokens per sec: 79350.28874332545, Loss: 2.2572131156921387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9987 , TFLOPS: 100.01287312329721, Tokens per sec: 81723.09318943966, Loss: 2.2703027725219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9988 , TFLOPS: 98.79221970597258, Tokens per sec: 80725.66585972933, Loss: 2.2432518005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9989 , TFLOPS: 100.5817602038405, Tokens per sec: 82187.94546740784, Loss: 2.2639193534851074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9990 , TFLOPS: 98.5352056990994, Tokens per sec: 80515.65309858412, Loss: 2.2534680366516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9991 , TFLOPS: 100.72343019203069, Tokens per sec: 82303.70766166798, Loss: 2.262878894805908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9992 , TFLOPS: 98.87505845901953, Tokens per sec: 80793.3555373032, Loss: 2.253577709197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9993 , TFLOPS: 100.2318248824348, Tokens per sec: 81902.00430815067, Loss: 2.267683982849121 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9994 , TFLOPS: 98.81366362105673, Tokens per sec: 80743.18823476006, Loss: 2.2704179286956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9995 , TFLOPS: 99.32145367507518, Tokens per sec: 81158.11655957741, Loss: 2.2774064540863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9996 , TFLOPS: 98.92003350743502, Tokens per sec: 80830.10580712424, Loss: 2.248286008834839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9997 , TFLOPS: 100.28167476226191, Tokens per sec: 81942.73792820748, Loss: 2.276366949081421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9998 , TFLOPS: 101.32199003129406, Tokens per sec: 82792.8063146311, Loss: 2.2805862426757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9999 , TFLOPS: 99.44590758678883, Tokens per sec: 81259.81105456734, Loss: 2.252934455871582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10000 , TFLOPS: 100.6835418600608, Tokens per sec: 82271.11387879837, Loss: 2.2650644779205322 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/model.pt +[rank0]:[2024-08-29 00:13:30,245] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007734132999758003, 'preprocessing_with_comm': 0.002192087998992065, 'state_converting': 2.3678293230004783, : 2.3794235339992156}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000, took 14.58s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008000 + eval ppl=7.1011786460876465, eval loss=1.9602607488632202 +------------------------------------------------------------------ +iteration: 10001 , TFLOPS: 99.09231885057865, Tokens per sec: 80970.88459602618, Loss: 2.2555153369903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10002 , TFLOPS: 97.35009084963266, Tokens per sec: 79547.26524751459, Loss: 2.258157730102539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10003 , TFLOPS: 100.48748680154418, Tokens per sec: 82110.91224357825, Loss: 2.2797751426696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10004 , TFLOPS: 98.29953143269903, Tokens per sec: 80323.07758871304, Loss: 2.2722713947296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10005 , TFLOPS: 100.12874736083776, Tokens per sec: 81817.77701180227, Loss: 2.233595132827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10006 , TFLOPS: 98.87735370536502, Tokens per sec: 80795.231042177, Loss: 2.2595481872558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10007 , TFLOPS: 100.12214441232706, Tokens per sec: 81812.38157259917, Loss: 2.2466273307800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10008 , TFLOPS: 100.61439534932863, Tokens per sec: 82214.61248488959, Loss: 2.262467861175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10009 , TFLOPS: 100.69889177742755, Tokens per sec: 82283.65669142075, Loss: 2.265000104904175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10010 , TFLOPS: 99.43016458366404, Tokens per sec: 81246.94704145312, Loss: 2.274998903274536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10011 , TFLOPS: 101.36678911769583, Tokens per sec: 82829.41280136106, Loss: 2.266019582748413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10012 , TFLOPS: 100.80413408616381, Tokens per sec: 82369.6528910672, Loss: 2.239806890487671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10013 , TFLOPS: 100.02574167427446, Tokens per sec: 81733.60841370914, Loss: 2.251502275466919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10014 , TFLOPS: 99.62586631812104, Tokens per sec: 81406.85996648876, Loss: 2.254594087600708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10015 , TFLOPS: 100.64082495326754, Tokens per sec: 82236.20879462658, Loss: 2.2547848224639893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10016 , TFLOPS: 100.03495662713921, Tokens per sec: 81741.13818890891, Loss: 2.2669243812561035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10017 , TFLOPS: 99.89755504622892, Tokens per sec: 81628.86382011576, Loss: 2.2721660137176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10018 , TFLOPS: 101.32012433367304, Tokens per sec: 82791.28180507764, Loss: 2.2637276649475098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10019 , TFLOPS: 99.31553176655848, Tokens per sec: 81153.27761568494, Loss: 2.262160301208496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10020 , TFLOPS: 100.5336366672569, Tokens per sec: 82148.62248685534, Loss: 2.2661356925964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10021 , TFLOPS: 99.91294652044007, Tokens per sec: 81641.44058990548, Loss: 2.276982307434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10022 , TFLOPS: 99.84355968068301, Tokens per sec: 81584.74281696384, Loss: 2.2402050495147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10023 , TFLOPS: 100.68253119453028, Tokens per sec: 82270.28803797654, Loss: 2.254608631134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10024 , TFLOPS: 99.11956217933616, Tokens per sec: 80993.1458212596, Loss: 2.2471532821655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10025 , TFLOPS: 100.57643335006433, Tokens per sec: 82183.59275806192, Loss: 2.265669345855713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10026 , TFLOPS: 99.57420211633269, Tokens per sec: 81364.64381725263, Loss: 2.2801156044006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10027 , TFLOPS: 100.14311356603915, Tokens per sec: 81829.51600789135, Loss: 2.2395572662353516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10028 , TFLOPS: 100.468838042104, Tokens per sec: 82095.67386217791, Loss: 2.2759451866149902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10029 , TFLOPS: 99.34724343383091, Tokens per sec: 81179.19003534417, Loss: 2.242523670196533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10030 , TFLOPS: 99.14520831594545, Tokens per sec: 81014.10193967336, Loss: 2.2531073093414307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10031 , TFLOPS: 100.62134853705966, Tokens per sec: 82220.29411357557, Loss: 2.2559499740600586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10032 , TFLOPS: 99.93455438970614, Tokens per sec: 81659.09693611882, Loss: 2.2699098587036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10033 , TFLOPS: 98.73471782202382, Tokens per sec: 80678.67959012461, Loss: 2.2493045330047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10034 , TFLOPS: 99.87321106817446, Tokens per sec: 81608.9717289777, Loss: 2.259831666946411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10035 , TFLOPS: 98.91922706188373, Tokens per sec: 80829.44683970453, Loss: 2.249377489089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10036 , TFLOPS: 100.62759751191616, Tokens per sec: 82225.40031179374, Loss: 2.256716728210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10037 , TFLOPS: 98.20939708334629, Tokens per sec: 80249.42649159236, Loss: 2.2746613025665283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10038 , TFLOPS: 100.12454300835051, Tokens per sec: 81814.3415271553, Loss: 2.2630062103271484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10039 , TFLOPS: 99.05457507356188, Tokens per sec: 80940.04318421452, Loss: 2.265620470046997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10040 , TFLOPS: 98.2219640973886, Tokens per sec: 80259.69532227004, Loss: 2.2601780891418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10041 , TFLOPS: 100.0237247866566, Tokens per sec: 81731.96036291728, Loss: 2.247342586517334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10042 , TFLOPS: 99.84352624723532, Tokens per sec: 81584.71549763308, Loss: 2.2469918727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10043 , TFLOPS: 99.33987123452884, Tokens per sec: 81173.16602151717, Loss: 2.269700765609741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10044 , TFLOPS: 100.20253495453663, Tokens per sec: 81878.07075407528, Loss: 2.256023406982422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10045 , TFLOPS: 100.139324633628, Tokens per sec: 81826.41997368241, Loss: 2.26240873336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10046 , TFLOPS: 101.30015241197322, Tokens per sec: 82774.9622337337, Loss: 2.2762556076049805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10047 , TFLOPS: 101.30195393386167, Tokens per sec: 82776.43430364407, Loss: 2.2391793727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10048 , TFLOPS: 99.31221072570925, Tokens per sec: 81150.56390771584, Loss: 2.2591116428375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10049 , TFLOPS: 99.9816971973438, Tokens per sec: 81697.6185378034, Loss: 2.2580556869506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10050 , TFLOPS: 100.39370707984054, Tokens per sec: 82034.28241886939, Loss: 2.2834959030151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10051 , TFLOPS: 99.90103468786893, Tokens per sec: 81631.70712487376, Loss: 2.258108615875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10052 , TFLOPS: 100.0819040126313, Tokens per sec: 81779.50010612771, Loss: 2.2380788326263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10053 , TFLOPS: 99.48382558921384, Tokens per sec: 81290.79483044503, Loss: 2.248159646987915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10054 , TFLOPS: 99.90868578191113, Tokens per sec: 81637.95903077233, Loss: 2.2864484786987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10055 , TFLOPS: 99.36503316756917, Tokens per sec: 81193.7264847302, Loss: 2.2605719566345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10056 , TFLOPS: 101.31964777501791, Tokens per sec: 82790.89239673282, Loss: 2.251469373703003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10057 , TFLOPS: 98.60326942722153, Tokens per sec: 80571.26972294896, Loss: 2.2665932178497314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10058 , TFLOPS: 100.65743601021488, Tokens per sec: 82249.78211686491, Loss: 2.2534446716308594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10059 , TFLOPS: 100.07316548669392, Tokens per sec: 81772.35963163462, Loss: 2.2685446739196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10060 , TFLOPS: 99.96315744425162, Tokens per sec: 81682.46922829608, Loss: 2.2711310386657715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10061 , TFLOPS: 100.7187021313717, Tokens per sec: 82299.84424159241, Loss: 2.2418277263641357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10062 , TFLOPS: 98.42521338412702, Tokens per sec: 80425.77554656612, Loss: 2.2557473182678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10063 , TFLOPS: 100.74712124776418, Tokens per sec: 82323.06623316985, Loss: 2.253969669342041 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10064 , TFLOPS: 98.1705684137043, Tokens per sec: 80217.69858608901, Loss: 2.2545275688171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10065 , TFLOPS: 100.17987942088115, Tokens per sec: 81859.55833432006, Loss: 2.263401508331299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10066 , TFLOPS: 100.13450518852748, Tokens per sec: 81822.48188102723, Loss: 2.25777530670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10067 , TFLOPS: 100.02140915849279, Tokens per sec: 81730.06820353505, Loss: 2.2522478103637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10068 , TFLOPS: 98.77374856931226, Tokens per sec: 80710.5726184748, Loss: 2.248260498046875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10069 , TFLOPS: 100.71682065957849, Tokens per sec: 82298.30684255679, Loss: 2.250392198562622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10070 , TFLOPS: 98.78631699497689, Tokens per sec: 80720.84260262545, Loss: 2.272885799407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10071 , TFLOPS: 98.94104649049419, Tokens per sec: 80847.2760565042, Loss: 2.2594423294067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10072 , TFLOPS: 99.97337447014641, Tokens per sec: 81690.81782316434, Loss: 2.2553935050964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10073 , TFLOPS: 98.49610019255464, Tokens per sec: 80483.6989825211, Loss: 2.25681209564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10074 , TFLOPS: 100.67927130654901, Tokens per sec: 82267.62429959005, Loss: 2.2571280002593994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10075 , TFLOPS: 98.14008021892717, Tokens per sec: 80192.78589730065, Loss: 2.276296854019165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10076 , TFLOPS: 100.15752486686836, Tokens per sec: 81841.29185277878, Loss: 2.251434087753296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10077 , TFLOPS: 99.00309844155073, Tokens per sec: 80897.98030307182, Loss: 2.2720086574554443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10078 , TFLOPS: 99.10245171501369, Tokens per sec: 80979.16442039917, Loss: 2.2415075302124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10079 , TFLOPS: 99.5934386127119, Tokens per sec: 81380.36245363556, Loss: 2.2909226417541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10080 , TFLOPS: 99.65112787597717, Tokens per sec: 81427.50183570334, Loss: 2.2665610313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10081 , TFLOPS: 99.59212848203828, Tokens per sec: 81379.29191213637, Loss: 2.268606424331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10082 , TFLOPS: 99.45846107579597, Tokens per sec: 81270.06883359062, Loss: 2.265920639038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10083 , TFLOPS: 100.52431434730616, Tokens per sec: 82141.00497924573, Loss: 2.2712206840515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10084 , TFLOPS: 101.24181229538388, Tokens per sec: 82727.29102266036, Loss: 2.283050537109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10085 , TFLOPS: 99.96879748328722, Tokens per sec: 81687.07784937958, Loss: 2.237537384033203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10086 , TFLOPS: 99.66829564830044, Tokens per sec: 81441.53006440648, Loss: 2.2550301551818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10087 , TFLOPS: 100.01677432988566, Tokens per sec: 81726.28096576907, Loss: 2.246232748031616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10088 , TFLOPS: 101.23989485547607, Tokens per sec: 82725.72423315239, Loss: 2.2616264820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10089 , TFLOPS: 99.31434032162315, Tokens per sec: 81152.30405535776, Loss: 2.285585880279541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10090 , TFLOPS: 99.95181747726747, Tokens per sec: 81673.2030493566, Loss: 2.2899885177612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10091 , TFLOPS: 99.84268049176129, Tokens per sec: 81584.02440906357, Loss: 2.2644710540771484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10092 , TFLOPS: 99.28765395610772, Tokens per sec: 81130.49793912694, Loss: 2.2663557529449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10093 , TFLOPS: 99.55644781090834, Tokens per sec: 81350.1363172539, Loss: 2.2483580112457275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10094 , TFLOPS: 100.69786332289281, Tokens per sec: 82282.81631474567, Loss: 2.268813133239746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10095 , TFLOPS: 98.46995671316851, Tokens per sec: 80462.33647252167, Loss: 2.265310525894165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10096 , TFLOPS: 99.9107425277339, Tokens per sec: 81639.63965072946, Loss: 2.267629623413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10097 , TFLOPS: 99.47577401584905, Tokens per sec: 81284.21568258274, Loss: 2.2688117027282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10098 , TFLOPS: 100.62699345883895, Tokens per sec: 82224.90672447464, Loss: 2.2575597763061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10099 , TFLOPS: 100.17940200226545, Tokens per sec: 81859.16822327931, Loss: 2.254075050354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10100 , TFLOPS: 98.28587796896981, Tokens per sec: 80311.92099202849, Loss: 2.256072521209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10101 , TFLOPS: 100.5146662877409, Tokens per sec: 82133.1213014118, Loss: 2.257598876953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10102 , TFLOPS: 99.02874316756659, Tokens per sec: 80918.93526885335, Loss: 2.284839153289795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10103 , TFLOPS: 100.15868491498658, Tokens per sec: 81842.23975795848, Loss: 2.2440013885498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10104 , TFLOPS: 100.0264192391987, Tokens per sec: 81734.1620694508, Loss: 2.268479585647583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10105 , TFLOPS: 98.81691559335415, Tokens per sec: 80745.8455050375, Loss: 2.256716251373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10106 , TFLOPS: 99.43940564510012, Tokens per sec: 81254.49815063864, Loss: 2.25896954536438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10107 , TFLOPS: 100.66350451743784, Tokens per sec: 82254.74085033406, Loss: 2.2620370388031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10108 , TFLOPS: 100.10149867055551, Tokens per sec: 81795.51140552897, Loss: 2.2622618675231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10109 , TFLOPS: 99.08737535934374, Tokens per sec: 80966.84514208135, Loss: 2.249722480773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10110 , TFLOPS: 99.9830220616479, Tokens per sec: 81698.70111853132, Loss: 2.2587180137634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10111 , TFLOPS: 98.99879439997406, Tokens per sec: 80894.46335990363, Loss: 2.2674148082733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10112 , TFLOPS: 99.97576925504575, Tokens per sec: 81692.77466355285, Loss: 2.270052194595337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10113 , TFLOPS: 98.37828143162797, Tokens per sec: 80387.42624004319, Loss: 2.2469117641448975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10114 , TFLOPS: 100.31804711509656, Tokens per sec: 81972.45871401635, Loss: 2.257840633392334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10115 , TFLOPS: 98.77113116955546, Tokens per sec: 80708.43387375567, Loss: 2.280501365661621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10116 , TFLOPS: 98.33520285639703, Tokens per sec: 80352.22562728082, Loss: 2.268916606903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10117 , TFLOPS: 100.17934089782271, Tokens per sec: 81859.11829326619, Loss: 2.2584261894226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10118 , TFLOPS: 99.02704244095672, Tokens per sec: 80917.54556135983, Loss: 2.2566211223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10119 , TFLOPS: 100.29016031887745, Tokens per sec: 81949.6716949557, Loss: 2.2770040035247803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10120 , TFLOPS: 99.3836936157008, Tokens per sec: 81208.97441726123, Loss: 2.2724101543426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10121 , TFLOPS: 99.99397199780492, Tokens per sec: 81707.64859324164, Loss: 2.2524049282073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10122 , TFLOPS: 101.30063223847414, Tokens per sec: 82775.35431231944, Loss: 2.260582208633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10123 , TFLOPS: 100.49715501839609, Tokens per sec: 82118.81239244998, Loss: 2.2818329334259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10124 , TFLOPS: 99.31358923762974, Tokens per sec: 81151.69032529212, Loss: 2.260634660720825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10125 , TFLOPS: 100.70563689828059, Tokens per sec: 82289.16830330464, Loss: 2.245607614517212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10126 , TFLOPS: 99.8500758801193, Tokens per sec: 81590.06737126512, Loss: 2.269266128540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10127 , TFLOPS: 100.61458218155136, Tokens per sec: 82214.76515030825, Loss: 2.2624151706695557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10128 , TFLOPS: 97.14019865155838, Tokens per sec: 79375.75692936225, Loss: 2.25467848777771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10129 , TFLOPS: 100.17995086435131, Tokens per sec: 81859.61671261863, Loss: 2.2756190299987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10130 , TFLOPS: 99.97716000589057, Tokens per sec: 81693.91108186926, Loss: 2.2630772590637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10131 , TFLOPS: 99.26452365370344, Tokens per sec: 81111.59757360544, Loss: 2.280972957611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10132 , TFLOPS: 100.70599416831618, Tokens per sec: 82289.46023784767, Loss: 2.2503502368927 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10133 , TFLOPS: 98.54632496971522, Tokens per sec: 80524.73894083979, Loss: 2.2594242095947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10134 , TFLOPS: 99.42996146657968, Tokens per sec: 81246.7810692548, Loss: 2.2893989086151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10135 , TFLOPS: 99.94513558975102, Tokens per sec: 81667.74310705982, Loss: 2.257606267929077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10136 , TFLOPS: 99.26102197433828, Tokens per sec: 81108.73626125492, Loss: 2.2714970111846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10137 , TFLOPS: 99.87593277377789, Tokens per sec: 81611.19570468922, Loss: 2.274648427963257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10138 , TFLOPS: 98.05687117792613, Tokens per sec: 80124.79364790743, Loss: 2.2510621547698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10139 , TFLOPS: 100.68869109491803, Tokens per sec: 82275.3214511533, Loss: 2.264906406402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10140 , TFLOPS: 99.42549938651013, Tokens per sec: 81243.1349887658, Loss: 2.2636258602142334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10141 , TFLOPS: 100.14072098160906, Tokens per sec: 81827.56096556292, Loss: 2.238776683807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10142 , TFLOPS: 100.62719447693163, Tokens per sec: 82225.07098153284, Loss: 2.259488821029663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10143 , TFLOPS: 100.05746358035354, Tokens per sec: 81759.52919976086, Loss: 2.25038480758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10144 , TFLOPS: 99.44645219813144, Tokens per sec: 81260.25607051492, Loss: 2.2788515090942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10145 , TFLOPS: 100.69303003399597, Tokens per sec: 82278.86690996813, Loss: 2.250718832015991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10146 , TFLOPS: 99.41942684413492, Tokens per sec: 81238.17295806987, Loss: 2.2804293632507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10147 , TFLOPS: 98.63133673343756, Tokens per sec: 80594.20424137481, Loss: 2.241974115371704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10148 , TFLOPS: 100.03895906569227, Tokens per sec: 81744.40868448257, Loss: 2.2578635215759277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10149 , TFLOPS: 98.89721418865027, Tokens per sec: 80811.45955432368, Loss: 2.27193546295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10150 , TFLOPS: 100.55320994481163, Tokens per sec: 82164.6163158063, Loss: 2.256654739379883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10151 , TFLOPS: 98.25986415489864, Tokens per sec: 80290.66443489578, Loss: 2.269721508026123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10152 , TFLOPS: 100.23977921507823, Tokens per sec: 81908.50399811637, Loss: 2.2624828815460205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10153 , TFLOPS: 98.39381152263917, Tokens per sec: 80400.11627718856, Loss: 2.293158531188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10154 , TFLOPS: 98.88078773185784, Tokens per sec: 80798.03707362399, Loss: 2.235434055328369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10155 , TFLOPS: 99.92597696595918, Tokens per sec: 81652.08810237267, Loss: 2.2547638416290283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10156 , TFLOPS: 99.06908487438147, Tokens per sec: 80951.8995159797, Loss: 2.2510907649993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10157 , TFLOPS: 100.11060070266973, Tokens per sec: 81802.94891027665, Loss: 2.242842197418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10158 , TFLOPS: 99.46505806358903, Tokens per sec: 81275.45940213792, Loss: 2.275968313217163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10159 , TFLOPS: 100.17324046603287, Tokens per sec: 81854.13347341162, Loss: 2.2580325603485107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10160 , TFLOPS: 101.23608623623196, Tokens per sec: 82722.61211232544, Loss: 2.258018732070923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10161 , TFLOPS: 101.28172522787814, Tokens per sec: 82759.90490725171, Loss: 2.2773680686950684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10162 , TFLOPS: 98.65779114153112, Tokens per sec: 80615.8208192251, Loss: 2.2407209873199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10163 , TFLOPS: 100.69063813293693, Tokens per sec: 82276.91242603977, Loss: 2.257560968399048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10164 , TFLOPS: 100.61935668486633, Tokens per sec: 82218.66651987356, Loss: 2.249940872192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10165 , TFLOPS: 99.26929283409741, Tokens per sec: 81115.49459367499, Loss: 2.2571146488189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10166 , TFLOPS: 101.2018291699154, Tokens per sec: 82694.61978158283, Loss: 2.272915840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10167 , TFLOPS: 99.89526049546434, Tokens per sec: 81626.98888361904, Loss: 2.2788302898406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10168 , TFLOPS: 99.31068757692834, Tokens per sec: 81149.31930363762, Loss: 2.2672343254089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10169 , TFLOPS: 98.88329011933585, Tokens per sec: 80800.08183884945, Loss: 2.258943796157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10170 , TFLOPS: 99.8788898208892, Tokens per sec: 81613.61198400504, Loss: 2.2689175605773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10171 , TFLOPS: 100.56747215394361, Tokens per sec: 82176.27033402996, Loss: 2.2659263610839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10172 , TFLOPS: 99.37726686208788, Tokens per sec: 81203.72295144474, Loss: 2.263692617416382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10173 , TFLOPS: 99.444776647744, Tokens per sec: 81258.88693516086, Loss: 2.2507264614105225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10174 , TFLOPS: 99.3248553018917, Tokens per sec: 81160.89611641486, Loss: 2.2662925720214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10175 , TFLOPS: 100.28835939976695, Tokens per sec: 81948.20011759072, Loss: 2.2484443187713623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10176 , TFLOPS: 98.91856595560812, Tokens per sec: 80828.90663274833, Loss: 2.2790074348449707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10177 , TFLOPS: 100.7017684153644, Tokens per sec: 82286.00726633021, Loss: 2.2619292736053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10178 , TFLOPS: 98.05135041346476, Tokens per sec: 80120.28248914893, Loss: 2.249770402908325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10179 , TFLOPS: 100.66020556190419, Tokens per sec: 82252.04518884486, Loss: 2.2662765979766846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10180 , TFLOPS: 100.7547066236441, Tokens per sec: 82329.26443906626, Loss: 2.2521352767944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10181 , TFLOPS: 99.45777404435606, Tokens per sec: 81269.50744251531, Loss: 2.2673444747924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10182 , TFLOPS: 100.61239805413366, Tokens per sec: 82212.9804435708, Loss: 2.2362375259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10183 , TFLOPS: 99.33324862332067, Tokens per sec: 81167.7545154178, Loss: 2.2627387046813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10184 , TFLOPS: 99.36163863872295, Tokens per sec: 81190.95272782701, Loss: 2.264989137649536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10185 , TFLOPS: 100.55327473036688, Tokens per sec: 82164.66925375123, Loss: 2.2649993896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10186 , TFLOPS: 99.41422495178439, Tokens per sec: 81233.92235792163, Loss: 2.2479586601257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10187 , TFLOPS: 99.98491130808519, Tokens per sec: 81700.24487042875, Loss: 2.2619173526763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10188 , TFLOPS: 98.78612609674454, Tokens per sec: 80720.68661476564, Loss: 2.2617039680480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10189 , TFLOPS: 99.4670293028977, Tokens per sec: 81277.07015252124, Loss: 2.2804205417633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10190 , TFLOPS: 100.16024075982827, Tokens per sec: 81843.51107882964, Loss: 2.276068687438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10191 , TFLOPS: 99.6662316363453, Tokens per sec: 81439.84350710553, Loss: 2.2807834148406982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10192 , TFLOPS: 100.10337486617532, Tokens per sec: 81797.04449326747, Loss: 2.268490791320801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10193 , TFLOPS: 100.06255915089896, Tokens per sec: 81763.69292162519, Loss: 2.260284185409546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10194 , TFLOPS: 100.11814490385015, Tokens per sec: 81809.11347126617, Loss: 2.2764904499053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10195 , TFLOPS: 100.53377479294333, Tokens per sec: 82148.7353529094, Loss: 2.23903489112854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10196 , TFLOPS: 100.19110804871029, Tokens per sec: 81868.73353516984, Loss: 2.2776801586151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10197 , TFLOPS: 98.39223863452358, Tokens per sec: 80398.83102981956, Loss: 2.2756948471069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10198 , TFLOPS: 101.3324517896983, Tokens per sec: 82801.35488673282, Loss: 2.27165150642395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10199 , TFLOPS: 100.18427555366452, Tokens per sec: 81863.15052758387, Loss: 2.2559752464294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10200 , TFLOPS: 100.09240430837484, Tokens per sec: 81788.08016808143, Loss: 2.2695882320404053 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/optimizer.pt +[rank0]:[2024-08-29 01:07:28,659] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007801608000590932, 'preprocessing_with_comm': 0.002177794998715399, 'state_converting': 2.3885894259983615, : 2.4002356409982895}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200, took 14.48s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008200 + eval ppl=7.871700763702393, eval loss=2.0632741451263428 +------------------------------------------------------------------ +iteration: 10201 , TFLOPS: 98.73330247016516, Tokens per sec: 80677.52306968649, Loss: 2.2626285552978516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10202 , TFLOPS: 97.60729467075794, Tokens per sec: 79757.43311077144, Loss: 2.2511727809906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10203 , TFLOPS: 100.13174548695923, Tokens per sec: 81820.22685783476, Loss: 2.2485125064849854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10204 , TFLOPS: 98.52585134201709, Tokens per sec: 80508.0094126096, Loss: 2.262984275817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10205 , TFLOPS: 98.42073325946626, Tokens per sec: 80422.11472137713, Loss: 2.2672529220581055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10206 , TFLOPS: 100.6655242465531, Tokens per sec: 82256.3912229866, Loss: 2.2631542682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10207 , TFLOPS: 100.23383799455851, Tokens per sec: 81903.64927388908, Loss: 2.2578816413879395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10208 , TFLOPS: 100.09278695380131, Tokens per sec: 81788.39283750956, Loss: 2.2785232067108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10209 , TFLOPS: 101.30435576654716, Tokens per sec: 82778.39690296006, Loss: 2.2561230659484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10210 , TFLOPS: 99.96134750775958, Tokens per sec: 81680.99028259653, Loss: 2.275582790374756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10211 , TFLOPS: 101.22704914933313, Tokens per sec: 82715.2276759851, Loss: 2.287795066833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10212 , TFLOPS: 100.04331732536401, Tokens per sec: 81747.96993065193, Loss: 2.261052131652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10213 , TFLOPS: 101.2399976298206, Tokens per sec: 82725.80821271495, Loss: 2.2647294998168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10214 , TFLOPS: 98.52856809554018, Tokens per sec: 80510.22934184877, Loss: 2.2635538578033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10215 , TFLOPS: 100.59043356882738, Tokens per sec: 82195.03269721089, Loss: 2.2278401851654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10216 , TFLOPS: 99.97687302277595, Tokens per sec: 81693.6765805787, Loss: 2.2799232006073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10217 , TFLOPS: 100.61057347077391, Tokens per sec: 82211.48952953861, Loss: 2.242488384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10218 , TFLOPS: 100.05676415202778, Tokens per sec: 81758.95767887101, Loss: 2.2460553646087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10219 , TFLOPS: 99.46170126753911, Tokens per sec: 81272.71647767391, Loss: 2.2746143341064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10220 , TFLOPS: 99.38457429939281, Tokens per sec: 81209.69404657677, Loss: 2.252702474594116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10221 , TFLOPS: 100.60805202826944, Tokens per sec: 82209.4291939605, Loss: 2.266374111175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10222 , TFLOPS: 100.04072865537557, Tokens per sec: 81745.85466176608, Loss: 2.2618961334228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10223 , TFLOPS: 100.69840953182076, Tokens per sec: 82283.26263612135, Loss: 2.2497241497039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10224 , TFLOPS: 99.39507302996053, Tokens per sec: 81218.27282958502, Loss: 2.2635645866394043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10225 , TFLOPS: 100.31600633238774, Tokens per sec: 81970.79113793054, Loss: 2.254479169845581 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10226 , TFLOPS: 99.42621867101025, Tokens per sec: 81243.72273464689, Loss: 2.244133949279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10227 , TFLOPS: 100.08976469938554, Tokens per sec: 81785.92327562669, Loss: 2.2596216201782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10228 , TFLOPS: 100.02883413321574, Tokens per sec: 81736.13534151703, Loss: 2.2598538398742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10229 , TFLOPS: 98.72492802136206, Tokens per sec: 80670.68009199199, Loss: 2.2383246421813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10230 , TFLOPS: 99.64697285031644, Tokens per sec: 81424.10665727609, Loss: 2.251715898513794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10231 , TFLOPS: 99.44637125226045, Tokens per sec: 81260.18992756002, Loss: 2.2379047870635986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10232 , TFLOPS: 99.5443248548658, Tokens per sec: 81340.2303377989, Loss: 2.2400877475738525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10233 , TFLOPS: 98.17889573209084, Tokens per sec: 80224.5030523069, Loss: 2.2682766914367676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10234 , TFLOPS: 100.59019719941152, Tokens per sec: 82194.8395536765, Loss: 2.257220506668091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10235 , TFLOPS: 97.80706036116659, Tokens per sec: 79920.6667988307, Loss: 2.27375864982605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10236 , TFLOPS: 100.00241420845714, Tokens per sec: 81714.54694088742, Loss: 2.2565042972564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10237 , TFLOPS: 97.87309866912162, Tokens per sec: 79974.62840023803, Loss: 2.2658090591430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10238 , TFLOPS: 100.15543821238751, Tokens per sec: 81839.58679368722, Loss: 2.2307729721069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10239 , TFLOPS: 97.39439099443038, Tokens per sec: 79583.464036216, Loss: 2.2427544593811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10240 , TFLOPS: 99.15722165242002, Tokens per sec: 81023.91834615752, Loss: 2.2576026916503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10241 , TFLOPS: 100.06552560061446, Tokens per sec: 81766.11688405093, Loss: 2.2826004028320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10242 , TFLOPS: 99.00175225102255, Tokens per sec: 80896.88029613721, Loss: 2.2559432983398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10243 , TFLOPS: 99.38988545515913, Tokens per sec: 81214.03392867475, Loss: 2.2649617195129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10244 , TFLOPS: 100.80337632348603, Tokens per sec: 82369.03370367666, Loss: 2.2304534912109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10245 , TFLOPS: 100.28771485691695, Tokens per sec: 81947.67344503598, Loss: 2.27544903755188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10246 , TFLOPS: 100.02057479819074, Tokens per sec: 81729.38642625391, Loss: 2.231065511703491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10247 , TFLOPS: 98.97248517921574, Tokens per sec: 80872.96541836241, Loss: 2.251446008682251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10248 , TFLOPS: 100.67726421937101, Tokens per sec: 82265.98425698989, Loss: 2.2601397037506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10249 , TFLOPS: 101.20428532657836, Tokens per sec: 82696.62677041924, Loss: 2.2491540908813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10250 , TFLOPS: 100.01056258236795, Tokens per sec: 81721.20518696775, Loss: 2.262817859649658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10251 , TFLOPS: 101.21921739010054, Tokens per sec: 82708.82814390873, Loss: 2.2591145038604736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10252 , TFLOPS: 98.77064934804933, Tokens per sec: 80708.04016499982, Loss: 2.2570250034332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10253 , TFLOPS: 100.65702312573983, Tokens per sec: 82249.44473833175, Loss: 2.254523515701294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10254 , TFLOPS: 100.61310118041492, Tokens per sec: 82213.5549861553, Loss: 2.2682371139526367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10255 , TFLOPS: 99.98509318856965, Tokens per sec: 81700.39348965463, Loss: 2.2925803661346436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10256 , TFLOPS: 99.21981176286053, Tokens per sec: 81075.06233661124, Loss: 2.28045392036438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10257 , TFLOPS: 99.48437417877192, Tokens per sec: 81291.24309709489, Loss: 2.2533702850341797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10258 , TFLOPS: 100.00466894388083, Tokens per sec: 81716.38934324417, Loss: 2.265781879425049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10259 , TFLOPS: 100.62061848394274, Tokens per sec: 82219.69756838049, Loss: 2.266160011291504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10260 , TFLOPS: 99.92067083050831, Tokens per sec: 81647.75232250369, Loss: 2.243786096572876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10261 , TFLOPS: 100.00243922169815, Tokens per sec: 81714.56737985054, Loss: 2.252157688140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10262 , TFLOPS: 99.3583842859496, Tokens per sec: 81188.2935124014, Loss: 2.2490031719207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10263 , TFLOPS: 100.70291586635699, Tokens per sec: 82286.94487807427, Loss: 2.2790918350219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10264 , TFLOPS: 99.9837688795083, Tokens per sec: 81699.31136262986, Loss: 2.2395753860473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10265 , TFLOPS: 99.46064236199328, Tokens per sec: 81271.85121869376, Loss: 2.283698320388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10266 , TFLOPS: 100.10289987401849, Tokens per sec: 81796.65636494875, Loss: 2.278262138366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10267 , TFLOPS: 98.68464171899345, Tokens per sec: 80637.76111726492, Loss: 2.256542682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10268 , TFLOPS: 99.05116521303472, Tokens per sec: 80937.25689940016, Loss: 2.2420811653137207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10269 , TFLOPS: 100.14816885993683, Tokens per sec: 81833.64681867, Loss: 2.2532856464385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10270 , TFLOPS: 99.55080916962547, Tokens per sec: 81345.52883831013, Loss: 2.2598483562469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10271 , TFLOPS: 98.41562801291582, Tokens per sec: 80417.94309300004, Loss: 2.2436699867248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10272 , TFLOPS: 100.55553256030069, Tokens per sec: 82166.51418471186, Loss: 2.2546372413635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10273 , TFLOPS: 98.74572834979125, Tokens per sec: 80687.67657579957, Loss: 2.262401580810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10274 , TFLOPS: 99.54540252722455, Tokens per sec: 81341.110931625, Loss: 2.280829906463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10275 , TFLOPS: 98.97656224167174, Tokens per sec: 80876.29689104814, Loss: 2.2609505653381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10276 , TFLOPS: 99.97007175021896, Tokens per sec: 81688.11908569206, Loss: 2.2386844158172607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10277 , TFLOPS: 98.35532539603139, Tokens per sec: 80368.66827241631, Loss: 2.264387369155884 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10278 , TFLOPS: 99.57249818217618, Tokens per sec: 81363.25148879016, Loss: 2.2710423469543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10279 , TFLOPS: 99.40772585976201, Tokens per sec: 81228.61178252863, Loss: 2.2585153579711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10280 , TFLOPS: 99.06492037955495, Tokens per sec: 80948.49660005332, Loss: 2.2643017768859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10281 , TFLOPS: 99.11413623020792, Tokens per sec: 80988.71213855057, Loss: 2.2534618377685547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10282 , TFLOPS: 100.65518031562871, Tokens per sec: 82247.9389307518, Loss: 2.272892713546753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10283 , TFLOPS: 99.86707308105379, Tokens per sec: 81603.95622169557, Loss: 2.2805421352386475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10284 , TFLOPS: 99.65774005649936, Tokens per sec: 81432.90481862071, Loss: 2.2852768898010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10285 , TFLOPS: 100.0651674037599, Tokens per sec: 81765.82419218025, Loss: 2.2563793659210205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10286 , TFLOPS: 100.02051272843111, Tokens per sec: 81729.3357074555, Loss: 2.260793685913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10287 , TFLOPS: 100.60914940738317, Tokens per sec: 82210.32589068338, Loss: 2.250608444213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10288 , TFLOPS: 99.48193282634091, Tokens per sec: 81289.24820517752, Loss: 2.277601480484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10289 , TFLOPS: 101.25128193320668, Tokens per sec: 82735.02890749554, Loss: 2.260185956954956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10290 , TFLOPS: 98.84957047233374, Tokens per sec: 80772.52864726308, Loss: 2.260438919067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10291 , TFLOPS: 100.63598753173882, Tokens per sec: 82232.25601296917, Loss: 2.2433762550354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10292 , TFLOPS: 99.3747245329112, Tokens per sec: 81201.64554882911, Loss: 2.2626802921295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10293 , TFLOPS: 101.1480128407147, Tokens per sec: 82650.64507363738, Loss: 2.246622085571289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10294 , TFLOPS: 100.10427558437812, Tokens per sec: 81797.78049329753, Loss: 2.2294540405273438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10295 , TFLOPS: 98.80379008195364, Tokens per sec: 80735.12031179141, Loss: 2.2586073875427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10296 , TFLOPS: 100.00832311661476, Tokens per sec: 81719.37526185183, Loss: 2.2559561729431152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10297 , TFLOPS: 99.41957488351214, Tokens per sec: 81238.29392485587, Loss: 2.2485954761505127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10298 , TFLOPS: 100.10934361072388, Tokens per sec: 81801.92170808726, Loss: 2.2700235843658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10299 , TFLOPS: 100.70350151685653, Tokens per sec: 82287.42342817344, Loss: 2.2685792446136475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10300 , TFLOPS: 99.0153180493902, Tokens per sec: 80907.96525919813, Loss: 2.261669635772705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10301 , TFLOPS: 101.3353633133163, Tokens per sec: 82803.73396763037, Loss: 2.268627405166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10302 , TFLOPS: 99.48838938707425, Tokens per sec: 81294.5240271594, Loss: 2.250467300415039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10303 , TFLOPS: 98.69045046637163, Tokens per sec: 80642.5075942781, Loss: 2.238743305206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10304 , TFLOPS: 100.07081821161617, Tokens per sec: 81770.44161274435, Loss: 2.269009828567505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10305 , TFLOPS: 99.4625640123971, Tokens per sec: 81273.42144870639, Loss: 2.267230749130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10306 , TFLOPS: 100.02785665318576, Tokens per sec: 81735.33661742197, Loss: 2.2698192596435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10307 , TFLOPS: 99.93156872579522, Tokens per sec: 81656.65727327939, Loss: 2.2668120861053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10308 , TFLOPS: 99.46279459446863, Tokens per sec: 81273.60986325276, Loss: 2.2659525871276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10309 , TFLOPS: 98.94619499919688, Tokens per sec: 80851.48303549956, Loss: 2.2427334785461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10310 , TFLOPS: 99.89333975166775, Tokens per sec: 81625.4193944185, Loss: 2.2691543102264404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10311 , TFLOPS: 98.80163689730053, Tokens per sec: 80733.36088918346, Loss: 2.2547898292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10312 , TFLOPS: 99.3339473569986, Tokens per sec: 81168.32546869304, Loss: 2.2506470680236816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10313 , TFLOPS: 97.87971492791092, Tokens per sec: 79980.03471561243, Loss: 2.248643159866333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10314 , TFLOPS: 101.19447100095564, Tokens per sec: 82688.6072323097, Loss: 2.287039279937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10315 , TFLOPS: 97.07678520165058, Tokens per sec: 79323.94016703517, Loss: 2.266430616378784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10316 , TFLOPS: 99.83981466281172, Tokens per sec: 81581.68266645588, Loss: 2.2551794052124023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10317 , TFLOPS: 100.03895947083234, Tokens per sec: 81744.40901553293, Loss: 2.27679181098938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10318 , TFLOPS: 98.07323982987141, Tokens per sec: 80138.16889477852, Loss: 2.250180244445801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10319 , TFLOPS: 100.2917537535207, Tokens per sec: 81950.97373142134, Loss: 2.264353036880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10320 , TFLOPS: 99.27992375245027, Tokens per sec: 81124.18139072528, Loss: 2.2696592807769775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10321 , TFLOPS: 101.29892024945934, Tokens per sec: 82773.95540202493, Loss: 2.262408971786499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10322 , TFLOPS: 99.47738888682785, Tokens per sec: 81285.53523522998, Loss: 2.285696268081665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10323 , TFLOPS: 100.60563595331672, Tokens per sec: 82207.45495692133, Loss: 2.264664649963379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10324 , TFLOPS: 99.31953612177873, Tokens per sec: 81156.54967741668, Loss: 2.276872158050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10325 , TFLOPS: 100.67235755978332, Tokens per sec: 82261.97489913214, Loss: 2.2712767124176025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10326 , TFLOPS: 99.45067370045595, Tokens per sec: 81263.70556873541, Loss: 2.2461960315704346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10327 , TFLOPS: 100.75677457670287, Tokens per sec: 82330.95421674443, Loss: 2.273789167404175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10328 , TFLOPS: 98.76028574604473, Tokens per sec: 80699.57179901902, Loss: 2.2672131061553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10329 , TFLOPS: 100.02439872267189, Tokens per sec: 81732.51105338403, Loss: 2.2519583702087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10330 , TFLOPS: 99.38958365364952, Tokens per sec: 81213.7873188922, Loss: 2.2483227252960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10331 , TFLOPS: 101.36930991371824, Tokens per sec: 82831.47260868203, Loss: 2.2680468559265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10332 , TFLOPS: 100.54355548180091, Tokens per sec: 82156.72740555224, Loss: 2.263310670852661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10333 , TFLOPS: 99.35286120795118, Tokens per sec: 81183.78046319222, Loss: 2.239698886871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10334 , TFLOPS: 99.95472085830639, Tokens per sec: 81675.57547674316, Loss: 2.2551956176757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10335 , TFLOPS: 100.55043506927014, Tokens per sec: 82162.3488935697, Loss: 2.266688346862793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10336 , TFLOPS: 99.93060447141092, Tokens per sec: 81655.86935619987, Loss: 2.2804665565490723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10337 , TFLOPS: 99.91223302726871, Tokens per sec: 81640.85757626823, Loss: 2.2303807735443115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10338 , TFLOPS: 98.98793346009779, Tokens per sec: 80885.58860634528, Loss: 2.261410713195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10339 , TFLOPS: 100.48741404459764, Tokens per sec: 82110.8527920043, Loss: 2.2599267959594727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10340 , TFLOPS: 99.42509237242358, Tokens per sec: 81242.8024070782, Loss: 2.244178056716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10341 , TFLOPS: 98.89850615465403, Tokens per sec: 80812.51525300353, Loss: 2.253492593765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10342 , TFLOPS: 100.14061714806694, Tokens per sec: 81827.47612050275, Loss: 2.228769302368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10343 , TFLOPS: 100.10379231429333, Tokens per sec: 81797.38560087078, Loss: 2.27516770362854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10344 , TFLOPS: 98.95815558449854, Tokens per sec: 80861.25633764241, Loss: 2.2625021934509277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10345 , TFLOPS: 99.18385151082222, Tokens per sec: 81045.6782889715, Loss: 2.2450413703918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10346 , TFLOPS: 99.96207006674227, Tokens per sec: 81681.5807041416, Loss: 2.2500178813934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10347 , TFLOPS: 98.7930588215031, Tokens per sec: 80726.35152263007, Loss: 2.2514123916625977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10348 , TFLOPS: 100.19419574210136, Tokens per sec: 81871.25656892399, Loss: 2.2521555423736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10349 , TFLOPS: 98.40115609322069, Tokens per sec: 80406.11771487679, Loss: 2.281956195831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10350 , TFLOPS: 99.34068372976863, Tokens per sec: 81173.82993229308, Loss: 2.2705256938934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10351 , TFLOPS: 98.26778557435615, Tokens per sec: 80297.13723064998, Loss: 2.2617342472076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10352 , TFLOPS: 100.6155155837509, Tokens per sec: 82215.52785727342, Loss: 2.248556613922119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10353 , TFLOPS: 97.80205949978445, Tokens per sec: 79916.58046626173, Loss: 2.2344436645507812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10354 , TFLOPS: 98.3007366642765, Tokens per sec: 80324.06241446042, Loss: 2.235999822616577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10355 , TFLOPS: 100.77600596918008, Tokens per sec: 82346.66868259765, Loss: 2.275599718093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10356 , TFLOPS: 99.0306526419508, Tokens per sec: 80920.49554952692, Loss: 2.262261152267456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10357 , TFLOPS: 99.46121622155826, Tokens per sec: 81272.32013411676, Loss: 2.244410514831543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10358 , TFLOPS: 100.12297323817172, Tokens per sec: 81813.05882753272, Loss: 2.273502826690674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10359 , TFLOPS: 100.05738047073508, Tokens per sec: 81759.4612887522, Loss: 2.233907699584961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10360 , TFLOPS: 100.48400893683272, Tokens per sec: 82108.07039079431, Loss: 2.237179756164551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10361 , TFLOPS: 99.9430183841685, Tokens per sec: 81666.01308387662, Loss: 2.2716164588928223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10362 , TFLOPS: 99.47479800278137, Tokens per sec: 81283.41815718032, Loss: 2.269160032272339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10363 , TFLOPS: 101.29125422697061, Tokens per sec: 82767.69129770843, Loss: 2.2612056732177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10364 , TFLOPS: 99.84104228478779, Tokens per sec: 81582.68578797432, Loss: 2.2490222454071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10365 , TFLOPS: 100.6938639813674, Tokens per sec: 82279.54834983304, Loss: 2.2637600898742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10366 , TFLOPS: 99.05152234042207, Tokens per sec: 80937.54871738164, Loss: 2.290315628051758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10367 , TFLOPS: 99.91470555548274, Tokens per sec: 81642.87794272043, Loss: 2.2618532180786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10368 , TFLOPS: 99.18150544363536, Tokens per sec: 81043.76125707997, Loss: 2.2537789344787598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10369 , TFLOPS: 99.50783749766477, Tokens per sec: 81310.41557896181, Loss: 2.2820510864257812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10370 , TFLOPS: 100.02989407301249, Tokens per sec: 81737.00144561022, Loss: 2.26301646232605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10371 , TFLOPS: 97.4196872304378, Tokens per sec: 79604.1342418407, Loss: 2.254427433013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10372 , TFLOPS: 99.404555408444, Tokens per sec: 81226.02112514201, Loss: 2.267332077026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10373 , TFLOPS: 99.94100302090135, Tokens per sec: 81664.36627867096, Loss: 2.257784843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10374 , TFLOPS: 100.61779361856657, Tokens per sec: 82217.38929816312, Loss: 2.240083932876587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10375 , TFLOPS: 100.75594854612, Tokens per sec: 82330.27924589124, Loss: 2.261112689971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10376 , TFLOPS: 98.99657084028702, Tokens per sec: 80892.64643204382, Loss: 2.2777624130249023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10377 , TFLOPS: 101.21573625777391, Tokens per sec: 82705.98362107233, Loss: 2.2766292095184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10378 , TFLOPS: 100.0714388448391, Tokens per sec: 81770.94874812732, Loss: 2.2591257095336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10379 , TFLOPS: 99.35432592968057, Tokens per sec: 81184.9773250226, Loss: 2.246551752090454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10380 , TFLOPS: 99.19829588389422, Tokens per sec: 81057.48115803987, Loss: 2.277402639389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10381 , TFLOPS: 99.42887370087452, Tokens per sec: 81245.89222789553, Loss: 2.227278470993042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10382 , TFLOPS: 99.3316271216982, Tokens per sec: 81166.42954470089, Loss: 2.263932466506958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10383 , TFLOPS: 100.00927630085368, Tokens per sec: 81720.15413323061, Loss: 2.2781131267547607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10384 , TFLOPS: 99.53137540543686, Tokens per sec: 81329.64900932202, Loss: 2.2694168090820312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10385 , TFLOPS: 99.27911859199797, Tokens per sec: 81123.52347339304, Loss: 2.2738351821899414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10386 , TFLOPS: 98.13187085817914, Tokens per sec: 80186.0778173057, Loss: 2.2661964893341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10387 , TFLOPS: 98.90004359987329, Tokens per sec: 80813.77153906954, Loss: 2.256638765335083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10388 , TFLOPS: 100.05958559398424, Tokens per sec: 81761.263151724, Loss: 2.2667222023010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10389 , TFLOPS: 97.20569669574718, Tokens per sec: 79429.27706733857, Loss: 2.250007152557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10390 , TFLOPS: 101.2154307309612, Tokens per sec: 82705.7339672487, Loss: 2.2944841384887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10391 , TFLOPS: 97.30756467730811, Tokens per sec: 79512.51601738714, Loss: 2.268435478210449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10392 , TFLOPS: 98.42376920769505, Tokens per sec: 80424.59547282713, Loss: 2.2829060554504395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10393 , TFLOPS: 100.76170891298355, Tokens per sec: 82334.98618994051, Loss: 2.2452759742736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10394 , TFLOPS: 99.06669257146187, Tokens per sec: 80949.94470368075, Loss: 2.2416718006134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10395 , TFLOPS: 100.0883603884375, Tokens per sec: 81784.77577700085, Loss: 2.261371612548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10396 , TFLOPS: 100.89727751934097, Tokens per sec: 82445.76278804155, Loss: 2.2508983612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10397 , TFLOPS: 100.58820153279446, Tokens per sec: 82193.20884311068, Loss: 2.267740488052368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10398 , TFLOPS: 100.41763219005179, Tokens per sec: 82053.83224231013, Loss: 2.2738845348358154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10399 , TFLOPS: 99.99334645900753, Tokens per sec: 81707.1374493875, Loss: 2.2728116512298584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10400 , TFLOPS: 99.34861937154538, Tokens per sec: 81180.31434947046, Loss: 2.263521432876587 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/model.pt +[rank0]:[2024-08-29 02:01:30,462] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007706796001002658, 'preprocessing_with_comm': 0.0016252519999397919, 'state_converting': 2.3878216939992853, : 2.39877579899985}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400, took 14.56s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008400 + eval ppl=7.433199882507324, eval loss=2.0059564113616943 +------------------------------------------------------------------ +iteration: 10401 , TFLOPS: 98.06276055245874, Tokens per sec: 80129.60600744402, Loss: 2.253959894180298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10402 , TFLOPS: 99.4989085223284, Tokens per sec: 81303.11948336218, Loss: 2.247575283050537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10403 , TFLOPS: 99.00029034934973, Tokens per sec: 80895.68573864765, Loss: 2.253669261932373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10404 , TFLOPS: 99.63110425599629, Tokens per sec: 81411.14002036313, Loss: 2.2620060443878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10405 , TFLOPS: 99.03914646494054, Tokens per sec: 80927.4360709423, Loss: 2.275066375732422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10406 , TFLOPS: 99.09522674830271, Tokens per sec: 80973.26071411246, Loss: 2.2717294692993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10407 , TFLOPS: 100.73122238302899, Tokens per sec: 82310.07486152153, Loss: 2.2506256103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10408 , TFLOPS: 99.48471690619694, Tokens per sec: 81291.52314849646, Loss: 2.253587245941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10409 , TFLOPS: 101.2606220888267, Tokens per sec: 82742.66098908964, Loss: 2.2402143478393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10410 , TFLOPS: 100.66954012099993, Tokens per sec: 82259.6726973749, Loss: 2.267137289047241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10411 , TFLOPS: 100.44152541495475, Tokens per sec: 82073.35601144467, Loss: 2.2608718872070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10412 , TFLOPS: 100.74409114540435, Tokens per sec: 82320.59025852985, Loss: 2.2586770057678223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10413 , TFLOPS: 99.86256237316739, Tokens per sec: 81600.27040616566, Loss: 2.275454521179199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10414 , TFLOPS: 99.78465837891481, Tokens per sec: 81536.61304703464, Loss: 2.254739761352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10415 , TFLOPS: 100.11119688036447, Tokens per sec: 81803.436062418, Loss: 2.2458624839782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10416 , TFLOPS: 99.93160136613416, Tokens per sec: 81656.68394454056, Loss: 2.2670865058898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10417 , TFLOPS: 100.03200057115667, Tokens per sec: 81738.72271946994, Loss: 2.276505470275879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10418 , TFLOPS: 100.52375984879028, Tokens per sec: 82140.55188423433, Loss: 2.252382278442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10419 , TFLOPS: 100.68246824367193, Tokens per sec: 82270.23659920969, Loss: 2.23476243019104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10420 , TFLOPS: 100.70917839430983, Tokens per sec: 82292.06215087634, Loss: 2.2659387588500977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10421 , TFLOPS: 99.96354965103305, Tokens per sec: 81682.78971055348, Loss: 2.284266710281372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10422 , TFLOPS: 99.30135637745089, Tokens per sec: 81141.69454033814, Loss: 2.2473061084747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10423 , TFLOPS: 100.65970580898374, Tokens per sec: 82251.63682786876, Loss: 2.265026569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10424 , TFLOPS: 100.04092266928564, Tokens per sec: 81746.01319552634, Loss: 2.246694326400757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10425 , TFLOPS: 99.99034585480403, Tokens per sec: 81704.6855784504, Loss: 2.248664379119873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10426 , TFLOPS: 98.76713668322544, Tokens per sec: 80705.16987614818, Loss: 2.2488744258880615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10427 , TFLOPS: 100.07458755325769, Tokens per sec: 81773.52163883112, Loss: 2.2565577030181885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10428 , TFLOPS: 100.12206895564825, Tokens per sec: 81812.31991500447, Loss: 2.250018358230591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10429 , TFLOPS: 100.64026815934206, Tokens per sec: 82235.75382397692, Loss: 2.270353317260742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10430 , TFLOPS: 98.22455615566805, Tokens per sec: 80261.8133598157, Loss: 2.259429931640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10431 , TFLOPS: 100.03096072213144, Tokens per sec: 81737.87303206332, Loss: 2.2626538276672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10432 , TFLOPS: 99.22202207466596, Tokens per sec: 81076.86843928593, Loss: 2.2600741386413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10433 , TFLOPS: 99.52264986462507, Tokens per sec: 81322.51914530931, Loss: 2.245364189147949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10434 , TFLOPS: 99.39951701408434, Tokens per sec: 81221.90412341092, Loss: 2.2537055015563965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10435 , TFLOPS: 97.54720503209816, Tokens per sec: 79708.33232017762, Loss: 2.2789320945739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10436 , TFLOPS: 100.6031102387266, Tokens per sec: 82205.39113051207, Loss: 2.2480311393737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10437 , TFLOPS: 97.93689224670064, Tokens per sec: 80026.75577467063, Loss: 2.2656755447387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10438 , TFLOPS: 99.9519771115317, Tokens per sec: 81673.33349062325, Loss: 2.253598213195801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10439 , TFLOPS: 97.89934399805921, Tokens per sec: 79996.0741341278, Loss: 2.265838146209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10440 , TFLOPS: 99.53409689870057, Tokens per sec: 81331.87281152527, Loss: 2.2523105144500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10441 , TFLOPS: 98.66715672791736, Tokens per sec: 80623.47368095294, Loss: 2.276156187057495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10442 , TFLOPS: 98.90798717849376, Tokens per sec: 80820.26244164623, Loss: 2.273667573928833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10443 , TFLOPS: 98.96075487957187, Tokens per sec: 80863.38028855863, Loss: 2.2893123626708984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10444 , TFLOPS: 98.33603309404819, Tokens per sec: 80352.9040358378, Loss: 2.243281602859497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10445 , TFLOPS: 100.01468061507595, Tokens per sec: 81724.57013750104, Loss: 2.2537076473236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10446 , TFLOPS: 99.68165391141287, Tokens per sec: 81452.44544506792, Loss: 2.2635912895202637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10447 , TFLOPS: 101.3276924904077, Tokens per sec: 82797.46594076722, Loss: 2.2746808528900146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10448 , TFLOPS: 100.54857383205828, Tokens per sec: 82160.82802873138, Loss: 2.252502202987671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10449 , TFLOPS: 101.12409838957038, Tokens per sec: 82631.10395999462, Loss: 2.2562971115112305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10450 , TFLOPS: 100.67097744538513, Tokens per sec: 82260.84717213007, Loss: 2.2564949989318848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10451 , TFLOPS: 99.94875123091609, Tokens per sec: 81670.69754053069, Loss: 2.2683565616607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10452 , TFLOPS: 99.31111594627473, Tokens per sec: 81149.66933525774, Loss: 2.265484571456909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10453 , TFLOPS: 99.31386026861298, Tokens per sec: 81151.91179168549, Loss: 2.247097969055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10454 , TFLOPS: 99.95031371351537, Tokens per sec: 81671.97428528445, Loss: 2.264082908630371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10455 , TFLOPS: 100.05363904143371, Tokens per sec: 81756.40407055708, Loss: 2.253683567047119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10456 , TFLOPS: 101.29279651492533, Tokens per sec: 82768.95154089824, Loss: 2.2608017921447754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10457 , TFLOPS: 100.58271499355781, Tokens per sec: 82188.72565066433, Loss: 2.2851359844207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10458 , TFLOPS: 100.61770992861521, Tokens per sec: 82217.3209129495, Loss: 2.248771905899048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10459 , TFLOPS: 99.34562997637059, Tokens per sec: 81177.87163771925, Loss: 2.2682647705078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10460 , TFLOPS: 100.00226021167634, Tokens per sec: 81714.42110615359, Loss: 2.2718915939331055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10461 , TFLOPS: 100.6766874466214, Tokens per sec: 82265.51296112873, Loss: 2.2453231811523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10462 , TFLOPS: 99.98273510716551, Tokens per sec: 81698.46664063689, Loss: 2.2578330039978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10463 , TFLOPS: 100.01747289393279, Tokens per sec: 81726.85178043462, Loss: 2.2551841735839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10464 , TFLOPS: 98.80605560799499, Tokens per sec: 80736.97153143963, Loss: 2.2455906867980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10465 , TFLOPS: 100.15473297804735, Tokens per sec: 81839.01052855352, Loss: 2.273688793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10466 , TFLOPS: 100.12507677163406, Tokens per sec: 81814.77767887455, Loss: 2.2686386108398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10467 , TFLOPS: 101.384546940166, Tokens per sec: 82843.92317522845, Loss: 2.281202793121338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10468 , TFLOPS: 97.57336098913399, Tokens per sec: 79729.70502598582, Loss: 2.2524900436401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10469 , TFLOPS: 99.88072622957041, Tokens per sec: 81615.11256081182, Loss: 2.2464091777801514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10470 , TFLOPS: 100.10550078739917, Tokens per sec: 81798.78163822556, Loss: 2.255342721939087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10471 , TFLOPS: 98.57692883037176, Tokens per sec: 80549.7461431957, Loss: 2.2614083290100098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10472 , TFLOPS: 100.14488841647763, Tokens per sec: 81830.96628387335, Loss: 2.2381362915039062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10473 , TFLOPS: 97.91583514688007, Tokens per sec: 80009.54947634928, Loss: 2.2349836826324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10474 , TFLOPS: 100.07664123006265, Tokens per sec: 81775.19975101479, Loss: 2.260171890258789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10475 , TFLOPS: 98.00592670263414, Tokens per sec: 80083.16560571888, Loss: 2.2711527347564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10476 , TFLOPS: 99.4087467231878, Tokens per sec: 81229.44595631321, Loss: 2.2695071697235107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10477 , TFLOPS: 98.67458697786034, Tokens per sec: 80629.545129453, Loss: 2.25325870513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10478 , TFLOPS: 99.47366264111244, Tokens per sec: 81282.49042393382, Loss: 2.233827829360962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10479 , TFLOPS: 98.97247564373103, Tokens per sec: 80872.95762667239, Loss: 2.252112865447998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10480 , TFLOPS: 99.03219301359375, Tokens per sec: 80921.75422684908, Loss: 2.2481846809387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10481 , TFLOPS: 98.98017627408431, Tokens per sec: 80879.25000996592, Loss: 2.2566423416137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10482 , TFLOPS: 98.44758520123213, Tokens per sec: 80444.0561342245, Loss: 2.2621848583221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10483 , TFLOPS: 100.08957279200777, Tokens per sec: 81785.76646316817, Loss: 2.2543530464172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10484 , TFLOPS: 98.95553881303059, Tokens per sec: 80859.11810631424, Loss: 2.2557883262634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10485 , TFLOPS: 100.75211944667267, Tokens per sec: 82327.15039016306, Loss: 2.2827980518341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10486 , TFLOPS: 100.61183036525978, Tokens per sec: 82212.51657037836, Loss: 2.2611148357391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10487 , TFLOPS: 101.24695928413512, Tokens per sec: 82731.49675966427, Loss: 2.240344285964966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10488 , TFLOPS: 100.59346545753193, Tokens per sec: 82197.51013151914, Loss: 2.2589166164398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10489 , TFLOPS: 99.37976065405121, Tokens per sec: 81205.7606930539, Loss: 2.220233678817749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10490 , TFLOPS: 100.71078123634133, Tokens per sec: 82293.37187436112, Loss: 2.260932445526123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10491 , TFLOPS: 100.1449067926909, Tokens per sec: 81830.98129955026, Loss: 2.2532429695129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10492 , TFLOPS: 100.05596247210525, Tokens per sec: 81758.30260556923, Loss: 2.2491493225097656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10493 , TFLOPS: 99.44682655146093, Tokens per sec: 81260.56196425717, Loss: 2.252049207687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10494 , TFLOPS: 99.98778131986123, Tokens per sec: 81702.59003093163, Loss: 2.2498228549957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10495 , TFLOPS: 101.23361659372709, Tokens per sec: 82720.59410386045, Loss: 2.277310848236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10496 , TFLOPS: 101.40029787940252, Tokens per sec: 82856.79367314387, Loss: 2.2750403881073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10497 , TFLOPS: 98.93595646854304, Tokens per sec: 80843.11686853909, Loss: 2.2610843181610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10498 , TFLOPS: 100.0051188555182, Tokens per sec: 81716.75697762481, Loss: 2.2583279609680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10499 , TFLOPS: 101.27749841530905, Tokens per sec: 82756.45106989366, Loss: 2.2808923721313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10500 , TFLOPS: 99.41977046012707, Tokens per sec: 81238.45373554251, Loss: 2.2817304134368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10501 , TFLOPS: 99.15216963249247, Tokens per sec: 81019.79021062415, Loss: 2.2481608390808105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10502 , TFLOPS: 98.99223243766546, Tokens per sec: 80889.10141158135, Loss: 2.2711551189422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10503 , TFLOPS: 100.6473445292994, Tokens per sec: 82241.53610803104, Loss: 2.2699365615844727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10504 , TFLOPS: 99.55218124008744, Tokens per sec: 81346.64999240484, Loss: 2.274705410003662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10505 , TFLOPS: 99.93759242670589, Tokens per sec: 81661.57939435737, Loss: 2.248582601547241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10506 , TFLOPS: 99.00156200260216, Tokens per sec: 80896.72483925548, Loss: 2.264690399169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10507 , TFLOPS: 99.33398840830556, Tokens per sec: 81168.3590127727, Loss: 2.2649245262145996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10508 , TFLOPS: 100.00420308483315, Tokens per sec: 81716.00867782404, Loss: 2.246997356414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10509 , TFLOPS: 98.80004970332621, Tokens per sec: 80732.06395212904, Loss: 2.292299270629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10510 , TFLOPS: 98.62098283306665, Tokens per sec: 80585.74380284864, Loss: 2.251596689224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10511 , TFLOPS: 98.47122912154671, Tokens per sec: 80463.3761901622, Loss: 2.2658309936523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10512 , TFLOPS: 99.38795097343288, Tokens per sec: 81212.45321385823, Loss: 2.244544267654419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10513 , TFLOPS: 98.49219069348553, Tokens per sec: 80480.50443019228, Loss: 2.256168842315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10514 , TFLOPS: 100.80588228216966, Tokens per sec: 82371.08138702616, Loss: 2.247312545776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10515 , TFLOPS: 98.7680229991653, Tokens per sec: 80705.89410771844, Loss: 2.2479896545410156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10516 , TFLOPS: 100.64828887378872, Tokens per sec: 82242.30775622284, Loss: 2.265876293182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10517 , TFLOPS: 98.80103239478174, Tokens per sec: 80732.86693461408, Loss: 2.258983612060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10518 , TFLOPS: 100.68420917268823, Tokens per sec: 82271.65915712439, Loss: 2.254401683807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10519 , TFLOPS: 99.67819248070674, Tokens per sec: 81449.6170209328, Loss: 2.2559680938720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10520 , TFLOPS: 98.91414278233941, Tokens per sec: 80825.29234400786, Loss: 2.2552273273468018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10521 , TFLOPS: 99.49106173260742, Tokens per sec: 81296.70766948683, Loss: 2.257601022720337 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10522 , TFLOPS: 98.27945238371825, Tokens per sec: 80306.67048091015, Loss: 2.2591209411621094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10523 , TFLOPS: 100.38145594313893, Tokens per sec: 82024.27169969819, Loss: 2.224961280822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10524 , TFLOPS: 100.63978214775592, Tokens per sec: 82235.35669139885, Loss: 2.2714686393737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10525 , TFLOPS: 100.09422302489806, Tokens per sec: 81789.56628817048, Loss: 2.2572484016418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10526 , TFLOPS: 100.61012309426155, Tokens per sec: 82211.12151529663, Loss: 2.291290283203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10527 , TFLOPS: 99.21643621017911, Tokens per sec: 81072.3040856208, Loss: 2.268406391143799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10528 , TFLOPS: 100.73027306914739, Tokens per sec: 82309.29915271136, Loss: 2.257842540740967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10529 , TFLOPS: 99.52074850565708, Tokens per sec: 81320.96549595124, Loss: 2.25886869430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10530 , TFLOPS: 99.95592802271547, Tokens per sec: 81676.56188185702, Loss: 2.2654452323913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10531 , TFLOPS: 100.10062217850567, Tokens per sec: 81794.79520131212, Loss: 2.279585361480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10532 , TFLOPS: 100.06592308251575, Tokens per sec: 81766.44167674457, Loss: 2.2637126445770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10533 , TFLOPS: 101.281833543196, Tokens per sec: 82759.9934144862, Loss: 2.2687761783599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10534 , TFLOPS: 100.64235883355832, Tokens per sec: 82237.46216769777, Loss: 2.2583706378936768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10535 , TFLOPS: 99.50593624828666, Tokens per sec: 81308.86201915244, Loss: 2.2563045024871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10536 , TFLOPS: 98.4488748810633, Tokens per sec: 80445.10996481386, Loss: 2.2482259273529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10537 , TFLOPS: 101.27609997938072, Tokens per sec: 82755.30837189768, Loss: 2.2560596466064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10538 , TFLOPS: 99.41697331366989, Tokens per sec: 81236.16811516738, Loss: 2.259216070175171 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10539 , TFLOPS: 100.06327665101792, Tokens per sec: 81764.27920944248, Loss: 2.240908145904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10540 , TFLOPS: 98.79613271363147, Tokens per sec: 80728.86327901717, Loss: 2.2367806434631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10541 , TFLOPS: 100.6482419340507, Tokens per sec: 82242.26940055458, Loss: 2.254371166229248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10542 , TFLOPS: 98.88772450486238, Tokens per sec: 80803.70528941441, Loss: 2.2595276832580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10543 , TFLOPS: 100.37705912440283, Tokens per sec: 82020.67894592475, Loss: 2.2690396308898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10544 , TFLOPS: 99.67291121005677, Tokens per sec: 81445.3015587327, Loss: 2.2717366218566895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10545 , TFLOPS: 98.81791010259738, Tokens per sec: 80746.6581441411, Loss: 2.2590255737304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10546 , TFLOPS: 100.72934803920263, Tokens per sec: 82308.54328693112, Loss: 2.257638454437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10547 , TFLOPS: 98.79684904849292, Tokens per sec: 80729.44861467257, Loss: 2.277632236480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10548 , TFLOPS: 99.55039719104998, Tokens per sec: 81345.19220001083, Loss: 2.2769134044647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10549 , TFLOPS: 98.92259525030434, Tokens per sec: 80832.1990731679, Loss: 2.258424758911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10550 , TFLOPS: 99.84971662851177, Tokens per sec: 81589.77381752865, Loss: 2.2728233337402344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10551 , TFLOPS: 98.01433519756137, Tokens per sec: 80090.03640337783, Loss: 2.2532148361206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10552 , TFLOPS: 100.6516151509108, Tokens per sec: 82245.0257428853, Loss: 2.2733378410339355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10553 , TFLOPS: 98.84546396498688, Tokens per sec: 80769.17311439906, Loss: 2.24127197265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10554 , TFLOPS: 99.96742249409029, Tokens per sec: 81685.9543103114, Loss: 2.264760732650757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10555 , TFLOPS: 99.46117248352071, Tokens per sec: 81272.28439464037, Loss: 2.2669119834899902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10556 , TFLOPS: 100.68730404182895, Tokens per sec: 82274.1880543683, Loss: 2.2659900188446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10557 , TFLOPS: 98.28678366935885, Tokens per sec: 80312.66106313116, Loss: 2.2554407119750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10558 , TFLOPS: 99.60376563081702, Tokens per sec: 81388.80092597034, Loss: 2.269167184829712 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10559 , TFLOPS: 98.94251791648236, Tokens per sec: 80848.478396557, Loss: 2.248809576034546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10560 , TFLOPS: 99.57643316162184, Tokens per sec: 81366.4668617906, Loss: 2.2504518032073975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10561 , TFLOPS: 99.97980779843284, Tokens per sec: 81696.0746613158, Loss: 2.240248680114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10562 , TFLOPS: 100.62771880522352, Tokens per sec: 82225.49942367755, Loss: 2.246574878692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10563 , TFLOPS: 100.08132510810582, Tokens per sec: 81779.02706833761, Loss: 2.272937059402466 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10564 , TFLOPS: 99.38397738186207, Tokens per sec: 81209.20628989638, Loss: 2.2741026878356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10565 , TFLOPS: 100.53889594011765, Tokens per sec: 82152.9199740956, Loss: 2.2499382495880127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10566 , TFLOPS: 99.34493448977643, Tokens per sec: 81177.30333771976, Loss: 2.235929489135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10567 , TFLOPS: 98.93520750621049, Tokens per sec: 80842.50487213716, Loss: 2.2555015087127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10568 , TFLOPS: 98.86623360113627, Tokens per sec: 80786.14451875474, Loss: 2.2616264820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10569 , TFLOPS: 100.71745182081193, Tokens per sec: 82298.82258064816, Loss: 2.23771333694458 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10570 , TFLOPS: 99.47453888051942, Tokens per sec: 81283.20642170968, Loss: 2.261115312576294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10571 , TFLOPS: 100.54496505250252, Tokens per sec: 82157.87920205812, Loss: 2.277165651321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10572 , TFLOPS: 100.07435066410667, Tokens per sec: 81773.32807060776, Loss: 2.2316770553588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10573 , TFLOPS: 100.72839574643032, Tokens per sec: 82307.76514399266, Loss: 2.2652361392974854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10574 , TFLOPS: 98.80316163473064, Tokens per sec: 80734.60679138784, Loss: 2.258026599884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10575 , TFLOPS: 99.62926356284504, Tokens per sec: 81409.6359426056, Loss: 2.2590510845184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10576 , TFLOPS: 101.27798192407315, Tokens per sec: 82756.84615735142, Loss: 2.244424343109131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10577 , TFLOPS: 97.66509440260164, Tokens per sec: 79804.6627595584, Loss: 2.244696855545044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10578 , TFLOPS: 99.58040623647946, Tokens per sec: 81369.71336353282, Loss: 2.2392942905426025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10579 , TFLOPS: 100.70177130170926, Tokens per sec: 82286.0096248369, Loss: 2.255666494369507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10580 , TFLOPS: 99.5245502475421, Tokens per sec: 81324.07199711102, Loss: 2.257290840148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10581 , TFLOPS: 101.30776343549745, Tokens per sec: 82781.18139698054, Loss: 2.2484724521636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10582 , TFLOPS: 98.54128074618052, Tokens per sec: 80520.61717593973, Loss: 2.2668583393096924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10583 , TFLOPS: 99.2380084516499, Tokens per sec: 81089.93131944549, Loss: 2.2546186447143555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10584 , TFLOPS: 100.0412003026422, Tokens per sec: 81746.24005688891, Loss: 2.252126693725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10585 , TFLOPS: 99.46204066573792, Tokens per sec: 81272.99380867904, Loss: 2.2630739212036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10586 , TFLOPS: 100.70078189506461, Tokens per sec: 82285.20115519795, Loss: 2.267335891723633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10587 , TFLOPS: 98.8707933781607, Tokens per sec: 80789.8704299405, Loss: 2.252915620803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10588 , TFLOPS: 100.56488477651124, Tokens per sec: 82174.15612132495, Loss: 2.2472572326660156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10589 , TFLOPS: 99.70119154202553, Tokens per sec: 81468.41014598488, Loss: 2.2535457611083984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10590 , TFLOPS: 100.83972394173497, Tokens per sec: 82398.73427821879, Loss: 2.2441041469573975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10591 , TFLOPS: 98.17119333508845, Tokens per sec: 80218.2092254389, Loss: 2.2628798484802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10592 , TFLOPS: 99.97769894715708, Tokens per sec: 81694.3514646517, Loss: 2.2494349479675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10593 , TFLOPS: 99.50409223471884, Tokens per sec: 81307.35522819703, Loss: 2.2363181114196777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10594 , TFLOPS: 101.25202324174536, Tokens per sec: 82735.63464978547, Loss: 2.250196933746338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10595 , TFLOPS: 98.94612138388865, Tokens per sec: 80851.4228825362, Loss: 2.2733542919158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10596 , TFLOPS: 99.03462869407358, Tokens per sec: 80923.74448406915, Loss: 2.266731023788452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10597 , TFLOPS: 100.70016656754227, Tokens per sec: 82284.69835523952, Loss: 2.2683072090148926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10598 , TFLOPS: 98.79106296995526, Tokens per sec: 80724.72066095253, Loss: 2.2661375999450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10599 , TFLOPS: 99.49319737210811, Tokens per sec: 81298.4527554991, Loss: 2.2473275661468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10600 , TFLOPS: 99.79836996234663, Tokens per sec: 81547.81713482433, Loss: 2.2459731101989746 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/model.pt +[rank0]:[2024-08-29 02:55:31,580] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00770293700043112, 'preprocessing_with_comm': 0.0017383710001013242, 'state_converting': 2.3820407499988505, : 2.3931188930000644}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600, took 14.30s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008600 + eval ppl=8.228888511657715, eval loss=2.1076509952545166 +------------------------------------------------------------------ +iteration: 10601 , TFLOPS: 99.33072158549697, Tokens per sec: 81165.6896077603, Loss: 2.2621936798095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10602 , TFLOPS: 98.05586267512972, Tokens per sec: 80123.96957431131, Loss: 2.2561709880828857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10603 , TFLOPS: 100.09996344666293, Tokens per sec: 81794.25693456615, Loss: 2.2538106441497803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10604 , TFLOPS: 99.31934778941437, Tokens per sec: 81156.39578619364, Loss: 2.2577362060546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10605 , TFLOPS: 99.70510860939015, Tokens per sec: 81471.61088256279, Loss: 2.246272087097168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10606 , TFLOPS: 100.05754640951201, Tokens per sec: 81759.59688159847, Loss: 2.267986536026001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10607 , TFLOPS: 100.30374388002446, Tokens per sec: 81960.77117244089, Loss: 2.275705575942993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10608 , TFLOPS: 100.2466834926467, Tokens per sec: 81914.14566104948, Loss: 2.2441177368164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10609 , TFLOPS: 99.98526992680733, Tokens per sec: 81700.53790701828, Loss: 2.2755675315856934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10610 , TFLOPS: 100.63273169762158, Tokens per sec: 82229.59558709933, Loss: 2.2422709465026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10611 , TFLOPS: 98.65469026389007, Tokens per sec: 80613.28701228091, Loss: 2.271501064300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10612 , TFLOPS: 100.61218541278046, Tokens per sec: 82212.80668884724, Loss: 2.2637219429016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10613 , TFLOPS: 100.30203823274083, Tokens per sec: 81959.37744413817, Loss: 2.26082444190979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10614 , TFLOPS: 101.18405580243015, Tokens per sec: 82680.09670548346, Loss: 2.2701098918914795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10615 , TFLOPS: 98.64517948317533, Tokens per sec: 80605.51550852961, Loss: 2.229386329650879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10616 , TFLOPS: 100.10471756147005, Tokens per sec: 81798.14164415686, Loss: 2.2540807723999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10617 , TFLOPS: 98.5803415424343, Tokens per sec: 80552.53475807304, Loss: 2.258547067642212 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10618 , TFLOPS: 100.06362138949295, Tokens per sec: 81764.56090412478, Loss: 2.2612812519073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10619 , TFLOPS: 100.1574706865914, Tokens per sec: 81841.24758067976, Loss: 2.242830276489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10620 , TFLOPS: 99.94447408130509, Tokens per sec: 81667.20257147991, Loss: 2.255746364593506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10621 , TFLOPS: 100.67852916937565, Tokens per sec: 82267.0178802014, Loss: 2.2619452476501465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10622 , TFLOPS: 99.41666814670799, Tokens per sec: 81235.91875538713, Loss: 2.2773187160491943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10623 , TFLOPS: 100.09247800441645, Tokens per sec: 81788.14038701414, Loss: 2.238011121749878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10624 , TFLOPS: 100.71800526141173, Tokens per sec: 82299.27481120909, Loss: 2.2436389923095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10625 , TFLOPS: 100.68374462572147, Tokens per sec: 82271.27956383936, Loss: 2.2513468265533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10626 , TFLOPS: 99.30326633795289, Tokens per sec: 81143.25521823109, Loss: 2.2625041007995605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10627 , TFLOPS: 99.90679765568056, Tokens per sec: 81636.41619422463, Loss: 2.245138168334961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10628 , TFLOPS: 99.40772560126862, Tokens per sec: 81228.61157130703, Loss: 2.280782461166382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10629 , TFLOPS: 99.275363720582, Tokens per sec: 81120.45527130007, Loss: 2.2564191818237305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10630 , TFLOPS: 100.13100817766326, Tokens per sec: 81819.624383429, Loss: 2.232362985610962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10631 , TFLOPS: 98.87173908424563, Tokens per sec: 80790.64319072715, Loss: 2.257077217102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10632 , TFLOPS: 100.00742183202203, Tokens per sec: 81718.63879900999, Loss: 2.26595139503479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10633 , TFLOPS: 98.8821347939105, Tokens per sec: 80799.13779270367, Loss: 2.2646589279174805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10634 , TFLOPS: 98.15163580079508, Tokens per sec: 80202.22826071207, Loss: 2.2556025981903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10635 , TFLOPS: 100.08659691190003, Tokens per sec: 81783.33479492618, Loss: 2.2568957805633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10636 , TFLOPS: 97.76168960344887, Tokens per sec: 79883.59318475235, Loss: 2.2590298652648926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10637 , TFLOPS: 100.0019999223635, Tokens per sec: 81714.20841705562, Loss: 2.2563183307647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10638 , TFLOPS: 97.79323299916166, Tokens per sec: 79909.36810538851, Loss: 2.2702994346618652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10639 , TFLOPS: 99.20851541774606, Tokens per sec: 81065.83180222442, Loss: 2.281888246536255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10640 , TFLOPS: 98.366253628229, Tokens per sec: 80377.5980122627, Loss: 2.249246120452881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10641 , TFLOPS: 100.0796893220748, Tokens per sec: 81777.6904254628, Loss: 2.2558603286743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10642 , TFLOPS: 99.65833857846896, Tokens per sec: 81433.39388632939, Loss: 2.2602810859680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10643 , TFLOPS: 100.29938217976213, Tokens per sec: 81957.2071148764, Loss: 2.2835817337036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10644 , TFLOPS: 99.42900651087838, Tokens per sec: 81246.0007503685, Loss: 2.2566211223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10645 , TFLOPS: 100.78569475808602, Tokens per sec: 82354.5856414243, Loss: 2.2498364448547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10646 , TFLOPS: 100.66085442199198, Tokens per sec: 82252.57538912573, Loss: 2.2668309211730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10647 , TFLOPS: 99.98471469089434, Tokens per sec: 81700.08420946072, Loss: 2.267536163330078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10648 , TFLOPS: 100.57815622412147, Tokens per sec: 82185.00056280472, Loss: 2.2731032371520996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10649 , TFLOPS: 100.10181804151728, Tokens per sec: 81795.77237176315, Loss: 2.2673499584198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10650 , TFLOPS: 100.69202227843466, Tokens per sec: 82278.04344695705, Loss: 2.2640182971954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10651 , TFLOPS: 100.61527162501824, Tokens per sec: 82215.3285123129, Loss: 2.2279231548309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10652 , TFLOPS: 100.6110809569243, Tokens per sec: 82211.90420953593, Loss: 2.2633302211761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10653 , TFLOPS: 98.66822448322304, Tokens per sec: 80624.34617129967, Loss: 2.2631757259368896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10654 , TFLOPS: 99.94098935391898, Tokens per sec: 81664.35511102785, Loss: 2.2431678771972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10655 , TFLOPS: 100.58288459485706, Tokens per sec: 82188.86423625187, Loss: 2.269322156906128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10656 , TFLOPS: 100.09342702801777, Tokens per sec: 81788.91585862881, Loss: 2.274247646331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10657 , TFLOPS: 100.1527567266769, Tokens per sec: 81837.3956826855, Loss: 2.262708902359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10658 , TFLOPS: 99.94423422435399, Tokens per sec: 81667.0065781907, Loss: 2.2642359733581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10659 , TFLOPS: 99.9912330388428, Tokens per sec: 81705.41051936663, Loss: 2.258042812347412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10660 , TFLOPS: 100.71385624140007, Tokens per sec: 82295.88454015227, Loss: 2.2427990436553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10661 , TFLOPS: 100.09319628313357, Tokens per sec: 81788.72731104403, Loss: 2.2572271823883057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10662 , TFLOPS: 100.73125804400144, Tokens per sec: 82310.10400102011, Loss: 2.2525362968444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10663 , TFLOPS: 100.1935906538887, Tokens per sec: 81870.76213576907, Loss: 2.260117530822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10664 , TFLOPS: 99.24160246768476, Tokens per sec: 81092.86808246576, Loss: 2.2573413848876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10665 , TFLOPS: 100.05094138281716, Tokens per sec: 81754.1997442576, Loss: 2.259486436843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10666 , TFLOPS: 99.36363571461575, Tokens per sec: 81192.58458994832, Loss: 2.2611067295074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10667 , TFLOPS: 99.32839013316307, Tokens per sec: 81163.78451804134, Loss: 2.251924991607666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10668 , TFLOPS: 100.07914367272984, Tokens per sec: 81777.24456133676, Loss: 2.2696127891540527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10669 , TFLOPS: 99.50451281473015, Tokens per sec: 81307.69889495097, Loss: 2.25186824798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10670 , TFLOPS: 100.14155304429924, Tokens per sec: 81828.24086540627, Loss: 2.24186372756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10671 , TFLOPS: 98.89233667873222, Tokens per sec: 80807.47401541138, Loss: 2.243354558944702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10672 , TFLOPS: 98.74959126308264, Tokens per sec: 80690.83306169021, Loss: 2.2571725845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10673 , TFLOPS: 99.94537497198009, Tokens per sec: 81667.93871244145, Loss: 2.2706003189086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10674 , TFLOPS: 99.50147023252174, Tokens per sec: 81305.21272270544, Loss: 2.26999831199646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10675 , TFLOPS: 100.05422520729734, Tokens per sec: 81756.8830417735, Loss: 2.2518739700317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10676 , TFLOPS: 98.85162431342849, Tokens per sec: 80774.20689369219, Loss: 2.268930196762085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10677 , TFLOPS: 99.55198725313382, Tokens per sec: 81346.49148067136, Loss: 2.2428700923919678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10678 , TFLOPS: 98.93455544017917, Tokens per sec: 80841.97205219729, Loss: 2.2525482177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10679 , TFLOPS: 100.1168790949412, Tokens per sec: 81808.07914622185, Loss: 2.275536298751831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10680 , TFLOPS: 99.35587736454312, Tokens per sec: 81186.24504238628, Loss: 2.2700870037078857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10681 , TFLOPS: 100.82753314751636, Tokens per sec: 82388.77286644379, Loss: 2.2502307891845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10682 , TFLOPS: 99.34034381327416, Tokens per sec: 81173.55217777523, Loss: 2.2823143005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10683 , TFLOPS: 100.64020436556484, Tokens per sec: 82235.70169643937, Loss: 2.2726802825927734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10684 , TFLOPS: 100.6789484548788, Tokens per sec: 82267.36048917938, Loss: 2.266486883163452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10685 , TFLOPS: 99.41179416857415, Tokens per sec: 81231.93610238662, Loss: 2.268143892288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10686 , TFLOPS: 100.65670385579189, Tokens per sec: 82249.18385463856, Loss: 2.2648732662200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10687 , TFLOPS: 100.01481174886045, Tokens per sec: 81724.67729029203, Loss: 2.2562479972839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10688 , TFLOPS: 100.6240075505161, Tokens per sec: 82222.46686191992, Loss: 2.253002405166626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10689 , TFLOPS: 100.67650501717479, Tokens per sec: 82265.36389333176, Loss: 2.248915195465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10690 , TFLOPS: 100.59790723575478, Tokens per sec: 82201.13962284656, Loss: 2.2535197734832764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10691 , TFLOPS: 98.70532307835553, Tokens per sec: 80654.66038838524, Loss: 2.2698745727539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10692 , TFLOPS: 100.54166868041058, Tokens per sec: 82155.18565156563, Loss: 2.2580554485321045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10693 , TFLOPS: 100.68904953914264, Tokens per sec: 82275.61434515641, Loss: 2.2753946781158447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10694 , TFLOPS: 98.85783826176375, Tokens per sec: 80779.28447083819, Loss: 2.250242233276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10695 , TFLOPS: 99.59410886941406, Tokens per sec: 81380.91013764082, Loss: 2.2724618911743164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10696 , TFLOPS: 100.60395696222658, Tokens per sec: 82206.08301008049, Loss: 2.2513256072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10697 , TFLOPS: 100.07847015906333, Tokens per sec: 81776.69421598206, Loss: 2.267660617828369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10698 , TFLOPS: 100.71352980710932, Tokens per sec: 82295.61780229014, Loss: 2.2578883171081543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10699 , TFLOPS: 100.03144429259943, Tokens per sec: 81738.26816994093, Loss: 2.2581779956817627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10700 , TFLOPS: 99.77303611885418, Tokens per sec: 81527.11619915551, Loss: 2.2629318237304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10701 , TFLOPS: 99.99223748280457, Tokens per sec: 81706.23127638447, Loss: 2.2773618698120117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10702 , TFLOPS: 98.95301297833294, Tokens per sec: 80857.05418176204, Loss: 2.2730236053466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10703 , TFLOPS: 101.36073637217017, Tokens per sec: 82824.46694718051, Loss: 2.267129421234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10704 , TFLOPS: 98.12482792723426, Tokens per sec: 80180.32285712947, Loss: 2.2406954765319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10705 , TFLOPS: 100.03353902719044, Tokens per sec: 81739.97983149851, Loss: 2.2497401237487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10706 , TFLOPS: 100.60203990648291, Tokens per sec: 82204.51653448294, Loss: 2.2503018379211426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10707 , TFLOPS: 98.98024593124102, Tokens per sec: 80879.30692862181, Loss: 2.247795820236206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10708 , TFLOPS: 101.32560926537252, Tokens per sec: 82795.76368396399, Loss: 2.2691760063171387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10709 , TFLOPS: 98.61846885125917, Tokens per sec: 80583.68956359809, Loss: 2.254448413848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10710 , TFLOPS: 98.63478643637582, Tokens per sec: 80597.02308244891, Loss: 2.2680206298828125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10711 , TFLOPS: 100.58885933965627, Tokens per sec: 82193.74635403091, Loss: 2.2423884868621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10712 , TFLOPS: 98.84014851036443, Tokens per sec: 80764.82971959509, Loss: 2.2894067764282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10713 , TFLOPS: 100.75366227870346, Tokens per sec: 82328.41107793144, Loss: 2.2604868412017822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10714 , TFLOPS: 99.55791175734026, Tokens per sec: 81351.33254556877, Loss: 2.2488842010498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10715 , TFLOPS: 99.44397158270527, Tokens per sec: 81258.22909579349, Loss: 2.2642810344696045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10716 , TFLOPS: 100.21881098089827, Tokens per sec: 81891.37031419756, Loss: 2.2590765953063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10717 , TFLOPS: 99.54875642112968, Tokens per sec: 81343.85148467179, Loss: 2.2745423316955566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10718 , TFLOPS: 99.4219699158934, Tokens per sec: 81240.25096746821, Loss: 2.2472972869873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10719 , TFLOPS: 100.08166887922536, Tokens per sec: 81779.30797256884, Loss: 2.255641460418701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10720 , TFLOPS: 100.1309093597821, Tokens per sec: 81819.5436367945, Loss: 2.243901252746582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10721 , TFLOPS: 100.56168943842027, Tokens per sec: 82171.54512831551, Loss: 2.2634096145629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10722 , TFLOPS: 100.70339081594098, Tokens per sec: 82287.33297160563, Loss: 2.2678582668304443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10723 , TFLOPS: 98.64426937161191, Tokens per sec: 80604.7718329428, Loss: 2.25443172454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10724 , TFLOPS: 101.34824628171548, Tokens per sec: 82814.26097274646, Loss: 2.268354892730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10725 , TFLOPS: 99.5186375036073, Tokens per sec: 81319.24054183433, Loss: 2.2690579891204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10726 , TFLOPS: 100.02232224724017, Tokens per sec: 81730.81431185555, Loss: 2.2428715229034424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10727 , TFLOPS: 100.04369510118974, Tokens per sec: 81748.27862100393, Loss: 2.265533447265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10728 , TFLOPS: 100.64009049200094, Tokens per sec: 82235.60864741894, Loss: 2.25211238861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10729 , TFLOPS: 98.69029936152164, Tokens per sec: 80642.38412261539, Loss: 2.250023365020752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10730 , TFLOPS: 100.59942783872393, Tokens per sec: 82202.38214667646, Loss: 2.2784740924835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10731 , TFLOPS: 100.64642615089898, Tokens per sec: 82240.78567739889, Loss: 2.279170036315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10732 , TFLOPS: 99.50415451615363, Tokens per sec: 81307.40611996068, Loss: 2.2689802646636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10733 , TFLOPS: 99.49411745759456, Tokens per sec: 81299.20458103518, Loss: 2.2381558418273926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10734 , TFLOPS: 100.09596962252888, Tokens per sec: 81790.99347805604, Loss: 2.262420654296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10735 , TFLOPS: 100.70276686685735, Tokens per sec: 82286.82312674753, Loss: 2.2590627670288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10736 , TFLOPS: 98.84047691394179, Tokens per sec: 80765.098066612, Loss: 2.259573221206665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10737 , TFLOPS: 100.18918913362172, Tokens per sec: 81867.1655402537, Loss: 2.2571945190429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10738 , TFLOPS: 100.49636942594358, Tokens per sec: 82118.17046463408, Loss: 2.2666378021240234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10739 , TFLOPS: 99.45805976159046, Tokens per sec: 81269.74090942256, Loss: 2.2519643306732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10740 , TFLOPS: 99.58166946010616, Tokens per sec: 81370.74557607646, Loss: 2.2535200119018555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10741 , TFLOPS: 100.86781927800469, Tokens per sec: 82421.69170072307, Loss: 2.2888107299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10742 , TFLOPS: 97.55300809883673, Tokens per sec: 79713.07415539393, Loss: 2.2569022178649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10743 , TFLOPS: 100.47186076301462, Tokens per sec: 82098.14380524572, Loss: 2.2404403686523438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10744 , TFLOPS: 101.23364638818676, Tokens per sec: 82720.61844968043, Loss: 2.264085292816162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10745 , TFLOPS: 99.59264546441018, Tokens per sec: 81379.71435174077, Loss: 2.2285573482513428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10746 , TFLOPS: 101.41625246436357, Tokens per sec: 82869.8305752231, Loss: 2.2577548027038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10747 , TFLOPS: 99.31067027343579, Tokens per sec: 81149.30516450842, Loss: 2.2638885974884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10748 , TFLOPS: 99.35280238021134, Tokens per sec: 81183.73239353164, Loss: 2.2836215496063232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10749 , TFLOPS: 100.74155726699702, Tokens per sec: 82318.51976125539, Loss: 2.250845193862915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10750 , TFLOPS: 98.32118924904584, Tokens per sec: 80340.77474797182, Loss: 2.2856876850128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10751 , TFLOPS: 100.73941033130768, Tokens per sec: 82316.76544483658, Loss: 2.260502338409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10752 , TFLOPS: 99.55075446483549, Tokens per sec: 81345.48413761803, Loss: 2.250312566757202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10753 , TFLOPS: 100.18431676071407, Tokens per sec: 81863.18419892482, Loss: 2.2675671577453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10754 , TFLOPS: 98.58188817731413, Tokens per sec: 80553.79855324728, Loss: 2.2478177547454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10755 , TFLOPS: 99.44416505038454, Tokens per sec: 81258.38718321452, Loss: 2.2533278465270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10756 , TFLOPS: 100.09990457956961, Tokens per sec: 81794.20883274883, Loss: 2.289698362350464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10757 , TFLOPS: 100.75954542445734, Tokens per sec: 82333.21834777258, Loss: 2.2880067825317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10758 , TFLOPS: 98.80693339259086, Tokens per sec: 80737.68879182912, Loss: 2.2552709579467773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10759 , TFLOPS: 100.66619525233932, Tokens per sec: 82256.93951908773, Loss: 2.250157117843628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10760 , TFLOPS: 101.33929161170504, Tokens per sec: 82806.9438813672, Loss: 2.2387309074401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10761 , TFLOPS: 98.05870107085377, Tokens per sec: 80126.28890052425, Loss: 2.2469124794006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10762 , TFLOPS: 101.29724652208736, Tokens per sec: 82772.58775630372, Loss: 2.2570016384124756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10763 , TFLOPS: 99.2512537629978, Tokens per sec: 81100.7544043125, Loss: 2.257572650909424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10764 , TFLOPS: 99.98727812831618, Tokens per sec: 81702.17886016687, Loss: 2.2428131103515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10765 , TFLOPS: 99.43553291926364, Tokens per sec: 81251.3336466648, Loss: 2.2537665367126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10766 , TFLOPS: 100.0942790336833, Tokens per sec: 81789.6120543907, Loss: 2.2386889457702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10767 , TFLOPS: 98.69984007411911, Tokens per sec: 80650.18008447858, Loss: 2.265683889389038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10768 , TFLOPS: 98.6705619986933, Tokens per sec: 80626.25621536344, Loss: 2.2665319442749023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10769 , TFLOPS: 100.72478293170501, Tokens per sec: 82304.81302007848, Loss: 2.2457284927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10770 , TFLOPS: 100.21378787649286, Tokens per sec: 81887.26580628168, Loss: 2.2567102909088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10771 , TFLOPS: 98.94920803423592, Tokens per sec: 80853.94506399239, Loss: 2.2809057235717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10772 , TFLOPS: 99.53333676742943, Tokens per sec: 81331.2516886961, Loss: 2.2523653507232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10773 , TFLOPS: 100.65819161823931, Tokens per sec: 82250.39954363275, Loss: 2.275268077850342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10774 , TFLOPS: 99.01821480409761, Tokens per sec: 80910.33227203866, Loss: 2.2768847942352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10775 , TFLOPS: 100.67910374108578, Tokens per sec: 82267.48737753653, Loss: 2.2618725299835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10776 , TFLOPS: 101.3183885225007, Tokens per sec: 82789.86342908496, Loss: 2.274355411529541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10777 , TFLOPS: 99.41085498604353, Tokens per sec: 81231.16867216415, Loss: 2.2472941875457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10778 , TFLOPS: 100.62213334165125, Tokens per sec: 82220.93539761002, Loss: 2.2598307132720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10779 , TFLOPS: 100.69765073691781, Tokens per sec: 82282.64260527302, Loss: 2.2836194038391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10780 , TFLOPS: 99.44333524862768, Tokens per sec: 81257.709130838, Loss: 2.2774786949157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10781 , TFLOPS: 99.8829139484471, Tokens per sec: 81616.90020222285, Loss: 2.264007568359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10782 , TFLOPS: 100.02406890993025, Tokens per sec: 81732.2415549027, Loss: 2.258728504180908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10783 , TFLOPS: 99.54757952499887, Tokens per sec: 81342.88981254729, Loss: 2.254079818725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10784 , TFLOPS: 101.22021904601675, Tokens per sec: 82709.64662274279, Loss: 2.2559192180633545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10785 , TFLOPS: 99.34253804124077, Tokens per sec: 81175.34513793119, Loss: 2.257997512817383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10786 , TFLOPS: 98.46774693015182, Tokens per sec: 80460.53080193386, Loss: 2.2618842124938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10787 , TFLOPS: 100.75901389801767, Tokens per sec: 82332.78402383601, Loss: 2.273571491241455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10788 , TFLOPS: 98.23642341245096, Tokens per sec: 80271.51040082338, Loss: 2.265730619430542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10789 , TFLOPS: 101.40289078698365, Tokens per sec: 82858.91240467578, Loss: 2.2547383308410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10790 , TFLOPS: 98.91801644618884, Tokens per sec: 80828.45761445622, Loss: 2.2575743198394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10791 , TFLOPS: 100.20739083814327, Tokens per sec: 81882.03862157182, Loss: 2.2621214389801025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10792 , TFLOPS: 100.09888250761024, Tokens per sec: 81793.37367144036, Loss: 2.2581541538238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10793 , TFLOPS: 99.47592063299685, Tokens per sec: 81284.33548722845, Loss: 2.2721025943756104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10794 , TFLOPS: 100.72402303793257, Tokens per sec: 82304.19209131562, Loss: 2.244285821914673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10795 , TFLOPS: 100.33229583471218, Tokens per sec: 81984.10170960893, Loss: 2.2749810218811035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10796 , TFLOPS: 97.54964836475602, Tokens per sec: 79710.32883018935, Loss: 2.247164726257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10797 , TFLOPS: 101.19562946537381, Tokens per sec: 82689.55384340732, Loss: 2.2396039962768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10798 , TFLOPS: 100.08428228958404, Tokens per sec: 81781.4434574481, Loss: 2.266064405441284 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10799 , TFLOPS: 99.36238313244084, Tokens per sec: 81191.56107280886, Loss: 2.2528610229492188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10800 , TFLOPS: 99.30650237091501, Tokens per sec: 81145.89946406733, Loss: 2.2553203105926514 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/optimizer.pt +[rank0]:[2024-08-29 03:49:28,555] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007703404997300822, 'preprocessing_with_comm': 0.001701880002656253, 'state_converting': 2.415451523997035, : 2.4264793889997236}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800, took 14.61s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0008800 + eval ppl=7.634856224060059, eval loss=2.032724142074585 +------------------------------------------------------------------ +iteration: 10801 , TFLOPS: 100.36913254382317, Tokens per sec: 82014.20193288477, Loss: 2.2787437438964844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10802 , TFLOPS: 97.96780206180387, Tokens per sec: 80052.01298028091, Loss: 2.272381544113159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10803 , TFLOPS: 99.09033604155094, Tokens per sec: 80969.2643917276, Loss: 2.262287139892578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10804 , TFLOPS: 100.7167781098729, Tokens per sec: 82298.27207409703, Loss: 2.2580161094665527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10805 , TFLOPS: 98.89476263384373, Tokens per sec: 80809.45632578262, Loss: 2.2610654830932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10806 , TFLOPS: 100.02609287777906, Tokens per sec: 81733.89539113354, Loss: 2.270807981491089 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10807 , TFLOPS: 100.6580447892582, Tokens per sec: 82250.27956589256, Loss: 2.259244680404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10808 , TFLOPS: 98.75241303532296, Tokens per sec: 80693.13880442665, Loss: 2.2759342193603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10809 , TFLOPS: 101.2892640262162, Tokens per sec: 82766.06505343967, Loss: 2.2655651569366455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10810 , TFLOPS: 99.36764332877375, Tokens per sec: 81195.85931464199, Loss: 2.247415065765381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10811 , TFLOPS: 101.18819047687275, Tokens per sec: 82683.47525440651, Loss: 2.263763904571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10812 , TFLOPS: 98.75459397697963, Tokens per sec: 80694.92090799675, Loss: 2.251803398132324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10813 , TFLOPS: 100.37978040994128, Tokens per sec: 82022.90257839033, Loss: 2.266879081726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10814 , TFLOPS: 100.19363296275093, Tokens per sec: 81870.79670742952, Loss: 2.2542312145233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10815 , TFLOPS: 99.24334811965544, Tokens per sec: 81094.29449962809, Loss: 2.261003017425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10816 , TFLOPS: 98.87776347501281, Tokens per sec: 80795.56587550462, Loss: 2.2424614429473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10817 , TFLOPS: 99.30251866417854, Tokens per sec: 81142.64427474323, Loss: 2.2645680904388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10818 , TFLOPS: 100.68102354562502, Tokens per sec: 82269.05609924572, Loss: 2.2421936988830566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10819 , TFLOPS: 99.28770443043788, Tokens per sec: 81130.53918300143, Loss: 2.25571346282959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10820 , TFLOPS: 100.69918814982226, Tokens per sec: 82283.89886493378, Loss: 2.2653255462646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10821 , TFLOPS: 99.34430165990558, Tokens per sec: 81176.7862361418, Loss: 2.240854263305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10822 , TFLOPS: 99.94179912720172, Tokens per sec: 81665.01679762261, Loss: 2.2586565017700195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10823 , TFLOPS: 100.78516622008497, Tokens per sec: 82354.15375941788, Loss: 2.231363296508789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10824 , TFLOPS: 100.20873433019393, Tokens per sec: 81883.13642351096, Loss: 2.2521965503692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10825 , TFLOPS: 100.1912188891144, Tokens per sec: 81868.82410571731, Loss: 2.2536497116088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10826 , TFLOPS: 99.64179557402238, Tokens per sec: 81419.87617154128, Loss: 2.2448158264160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10827 , TFLOPS: 99.72706543185775, Tokens per sec: 81489.55236741998, Loss: 2.2730159759521484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10828 , TFLOPS: 100.69441936033762, Tokens per sec: 82280.00216428631, Loss: 2.266266107559204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10829 , TFLOPS: 99.49019335804411, Tokens per sec: 81295.99809827721, Loss: 2.252077579498291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10830 , TFLOPS: 100.129237434543, Tokens per sec: 81818.17746364244, Loss: 2.249316692352295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10831 , TFLOPS: 99.47503692450633, Tokens per sec: 81283.61338627218, Loss: 2.2702131271362305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10832 , TFLOPS: 100.66208427530177, Tokens per sec: 82253.5803339244, Loss: 2.2625892162323 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10833 , TFLOPS: 100.1766129562592, Tokens per sec: 81856.8892219912, Loss: 2.2598447799682617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10834 , TFLOPS: 99.37990425106425, Tokens per sec: 81205.87802986977, Loss: 2.267219305038452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10835 , TFLOPS: 99.4184168419292, Tokens per sec: 81237.34765926782, Loss: 2.2684969902038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10836 , TFLOPS: 100.15209261069268, Tokens per sec: 81836.85301641891, Loss: 2.241819381713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10837 , TFLOPS: 99.93030306524108, Tokens per sec: 81655.6230694596, Loss: 2.232736349105835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10838 , TFLOPS: 99.97222803509948, Tokens per sec: 81689.88104157565, Loss: 2.280689239501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10839 , TFLOPS: 100.67332855059756, Tokens per sec: 82262.76832072198, Loss: 2.270822048187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10840 , TFLOPS: 97.70319351669218, Tokens per sec: 79835.79452644022, Loss: 2.2481563091278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10841 , TFLOPS: 100.15628484535195, Tokens per sec: 81840.27859927673, Loss: 2.2606098651885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10842 , TFLOPS: 99.78710455330624, Tokens per sec: 81538.6118791, Loss: 2.244253158569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10843 , TFLOPS: 100.13937994967631, Tokens per sec: 81826.46517384944, Loss: 2.2656497955322266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10844 , TFLOPS: 98.93876453975514, Tokens per sec: 80845.41141581268, Loss: 2.2868685722351074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10845 , TFLOPS: 100.67537028164602, Tokens per sec: 82264.43667172051, Loss: 2.251387357711792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10846 , TFLOPS: 99.44505518875563, Tokens per sec: 81259.11453819173, Loss: 2.2634544372558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10847 , TFLOPS: 99.87609083001193, Tokens per sec: 81611.32485650679, Loss: 2.2555270195007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10848 , TFLOPS: 100.67036782189764, Tokens per sec: 82260.34903308532, Loss: 2.2571873664855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10849 , TFLOPS: 100.78089814314795, Tokens per sec: 82350.66620388256, Loss: 2.281137704849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10850 , TFLOPS: 98.04216553312563, Tokens per sec: 80112.77728698397, Loss: 2.261167526245117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10851 , TFLOPS: 101.39439833123947, Tokens per sec: 82851.97300047196, Loss: 2.2426645755767822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10852 , TFLOPS: 100.34532035173213, Tokens per sec: 81994.74437774681, Loss: 2.2461962699890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10853 , TFLOPS: 100.01268293102405, Tokens per sec: 81722.93777843728, Loss: 2.266374111175537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10854 , TFLOPS: 98.17231505393549, Tokens per sec: 80219.12581078443, Loss: 2.25911545753479 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10855 , TFLOPS: 100.03251437133264, Tokens per sec: 81739.14255882011, Loss: 2.255640983581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10856 , TFLOPS: 100.64455812204068, Tokens per sec: 82239.25926293143, Loss: 2.256775140762329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10857 , TFLOPS: 100.00492950869653, Tokens per sec: 81716.60225746261, Loss: 2.2613911628723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10858 , TFLOPS: 99.73350989263125, Tokens per sec: 81494.81830221316, Loss: 2.2510151863098145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10859 , TFLOPS: 99.37538585835864, Tokens per sec: 81202.18593487625, Loss: 2.2560770511627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10860 , TFLOPS: 98.52206065993961, Tokens per sec: 80504.91194870324, Loss: 2.254411458969116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10861 , TFLOPS: 100.22717902936749, Tokens per sec: 81898.20806201448, Loss: 2.266101837158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10862 , TFLOPS: 100.19856385560138, Tokens per sec: 81874.82586691009, Loss: 2.2519962787628174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10863 , TFLOPS: 99.4618812186613, Tokens per sec: 81272.86352036819, Loss: 2.232001304626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10864 , TFLOPS: 99.49386286473543, Tokens per sec: 81298.99654665618, Loss: 2.248410940170288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10865 , TFLOPS: 100.68314525850587, Tokens per sec: 82270.78980545841, Loss: 2.2483773231506348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10866 , TFLOPS: 100.07651023971408, Tokens per sec: 81775.09271542898, Loss: 2.2590525150299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10867 , TFLOPS: 100.18313255901447, Tokens per sec: 81862.21655723201, Loss: 2.261159658432007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10868 , TFLOPS: 100.05488483996605, Tokens per sec: 81757.42204460756, Loss: 2.2490289211273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10869 , TFLOPS: 99.51855982658319, Tokens per sec: 81319.17706993835, Loss: 2.256998300552368 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10870 , TFLOPS: 100.01627899547267, Tokens per sec: 81725.87621526918, Loss: 2.262472629547119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10871 , TFLOPS: 100.03331403644545, Tokens per sec: 81739.795985769, Loss: 2.2445695400238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10872 , TFLOPS: 99.34118853511691, Tokens per sec: 81174.24242173802, Loss: 2.2595784664154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10873 , TFLOPS: 100.11342336517312, Tokens per sec: 81805.25538046996, Loss: 2.2547218799591064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10874 , TFLOPS: 99.49723997747384, Tokens per sec: 81301.75607240935, Loss: 2.265150547027588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10875 , TFLOPS: 100.24367934344052, Tokens per sec: 81911.69089339941, Loss: 2.2123095989227295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10876 , TFLOPS: 99.04316809767236, Tokens per sec: 80930.72225057353, Loss: 2.252957344055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10877 , TFLOPS: 99.97092554746786, Tokens per sec: 81688.81674540283, Loss: 2.2428805828094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10878 , TFLOPS: 98.88233295685347, Tokens per sec: 80799.29971674555, Loss: 2.2738678455352783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10879 , TFLOPS: 98.83501684751435, Tokens per sec: 80760.63650577953, Loss: 2.2702889442443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10880 , TFLOPS: 100.67435081658066, Tokens per sec: 82263.60364057224, Loss: 2.2754428386688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10881 , TFLOPS: 99.16045311582221, Tokens per sec: 81026.55885808871, Loss: 2.2768540382385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10882 , TFLOPS: 98.95971628690361, Tokens per sec: 80862.53162775363, Loss: 2.264589309692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10883 , TFLOPS: 100.68465427157422, Tokens per sec: 82272.022858882, Loss: 2.2626473903656006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10884 , TFLOPS: 100.75129928341164, Tokens per sec: 82326.48021364959, Loss: 2.2467634677886963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10885 , TFLOPS: 99.8507744336838, Tokens per sec: 81590.63817736504, Loss: 2.2475743293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10886 , TFLOPS: 100.71462716998782, Tokens per sec: 82296.5144857468, Loss: 2.259159564971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10887 , TFLOPS: 100.65941710905027, Tokens per sec: 82251.40092372132, Loss: 2.2562575340270996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10888 , TFLOPS: 99.8436609309613, Tokens per sec: 81584.82555117265, Loss: 2.269132614135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10889 , TFLOPS: 100.11212843155685, Tokens per sec: 81804.19725687755, Loss: 2.249065399169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10890 , TFLOPS: 100.74842394935064, Tokens per sec: 82324.13070417065, Loss: 2.249221086502075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10891 , TFLOPS: 98.02205569634557, Tokens per sec: 80096.34502167774, Loss: 2.2484195232391357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10892 , TFLOPS: 100.19017880946024, Tokens per sec: 81867.97422985788, Loss: 2.253518581390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10893 , TFLOPS: 99.12903871355762, Tokens per sec: 81000.88934131966, Loss: 2.2597155570983887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10894 , TFLOPS: 100.69004472019556, Tokens per sec: 82276.42753321299, Loss: 2.246718168258667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10895 , TFLOPS: 98.88470137055631, Tokens per sec: 80801.23500855053, Loss: 2.2864978313446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10896 , TFLOPS: 99.88742167290303, Tokens per sec: 81620.58357991494, Loss: 2.2600269317626953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10897 , TFLOPS: 100.10424138949209, Tokens per sec: 81797.7525517759, Loss: 2.2536609172821045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10898 , TFLOPS: 98.35229691588555, Tokens per sec: 80366.19362332913, Loss: 2.2867929935455322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10899 , TFLOPS: 100.08163457496143, Tokens per sec: 81779.27994167173, Loss: 2.2707040309906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10900 , TFLOPS: 100.77430663191694, Tokens per sec: 82345.28011037511, Loss: 2.26816463470459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10901 , TFLOPS: 98.44814194320894, Tokens per sec: 80444.51106242558, Loss: 2.2480380535125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10902 , TFLOPS: 100.11392863248352, Tokens per sec: 81805.66824739601, Loss: 2.2617709636688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10903 , TFLOPS: 99.96074411471054, Tokens per sec: 81680.49723460342, Loss: 2.2610175609588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10904 , TFLOPS: 98.82178685217876, Tokens per sec: 80749.82593602045, Loss: 2.2446680068969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10905 , TFLOPS: 101.31314526925495, Tokens per sec: 82785.57903188451, Loss: 2.2540390491485596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10906 , TFLOPS: 99.12761844771866, Tokens per sec: 80999.72880554174, Loss: 2.279132604598999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10907 , TFLOPS: 100.70528406478324, Tokens per sec: 82288.87999397122, Loss: 2.2540454864501953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10908 , TFLOPS: 99.4658681899413, Tokens per sec: 81276.12137723487, Loss: 2.2725703716278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10909 , TFLOPS: 100.18996020305264, Tokens per sec: 81867.7956009348, Loss: 2.2531566619873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10910 , TFLOPS: 98.81774582956994, Tokens per sec: 80746.5239124216, Loss: 2.2813608646392822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10911 , TFLOPS: 98.37555783068838, Tokens per sec: 80385.20071560374, Loss: 2.2731499671936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10912 , TFLOPS: 99.40126741977105, Tokens per sec: 81223.33442495698, Loss: 2.262723207473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10913 , TFLOPS: 99.64712651543914, Tokens per sec: 81424.23222100355, Loss: 2.2655680179595947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10914 , TFLOPS: 98.41562434527023, Tokens per sec: 80417.94009607242, Loss: 2.254500150680542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10915 , TFLOPS: 100.63551157147606, Tokens per sec: 82231.86709358614, Loss: 2.230668544769287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10916 , TFLOPS: 99.60566383587914, Tokens per sec: 81390.35199819073, Loss: 2.2622101306915283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10917 , TFLOPS: 98.97450684189624, Tokens per sec: 80874.61737098088, Loss: 2.248577117919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10918 , TFLOPS: 100.63315872542297, Tokens per sec: 82229.94452250852, Loss: 2.257378101348877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10919 , TFLOPS: 99.52941650219357, Tokens per sec: 81328.04833905528, Loss: 2.263028621673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10920 , TFLOPS: 99.69620826121934, Tokens per sec: 81464.33817895707, Loss: 2.2546753883361816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10921 , TFLOPS: 100.66067021352983, Tokens per sec: 82252.42486764934, Loss: 2.2679357528686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10922 , TFLOPS: 100.7079534412705, Tokens per sec: 82291.06121021483, Loss: 2.2623989582061768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10923 , TFLOPS: 98.69803183435825, Tokens per sec: 80648.70252522174, Loss: 2.2599129676818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10924 , TFLOPS: 100.04692503756873, Tokens per sec: 81750.91788516517, Loss: 2.2763285636901855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10925 , TFLOPS: 101.33192697269824, Tokens per sec: 82800.92604525208, Loss: 2.2417869567871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10926 , TFLOPS: 100.07436540299551, Tokens per sec: 81773.34011413324, Loss: 2.264799118041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10927 , TFLOPS: 99.34608001998822, Tokens per sec: 81178.23937994438, Loss: 2.2663655281066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10928 , TFLOPS: 99.98415614262461, Tokens per sec: 81699.62780529122, Loss: 2.256856679916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10929 , TFLOPS: 100.00284217567553, Tokens per sec: 81714.89664391842, Loss: 2.2503199577331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10930 , TFLOPS: 100.0208485614544, Tokens per sec: 81729.61012526395, Loss: 2.2717926502227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10931 , TFLOPS: 100.11672985812291, Tokens per sec: 81807.95720097593, Loss: 2.2580151557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10932 , TFLOPS: 101.28447527823552, Tokens per sec: 82762.15204419132, Loss: 2.2570488452911377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10933 , TFLOPS: 98.11260302115271, Tokens per sec: 80170.33357167314, Loss: 2.272145986557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10934 , TFLOPS: 101.21134481825005, Tokens per sec: 82702.39526279167, Loss: 2.280069589614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10935 , TFLOPS: 100.01580472981706, Tokens per sec: 81725.4886805933, Loss: 2.260923147201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10936 , TFLOPS: 97.6003297347381, Tokens per sec: 79751.74188225622, Loss: 2.2360363006591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10937 , TFLOPS: 100.17618898800825, Tokens per sec: 81856.54278661942, Loss: 2.254242181777954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10938 , TFLOPS: 100.79909516174808, Tokens per sec: 82365.53545621366, Loss: 2.266127586364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10939 , TFLOPS: 98.1032221165495, Tokens per sec: 80162.6681930359, Loss: 2.240184783935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10940 , TFLOPS: 100.12935013624599, Tokens per sec: 81818.26955510516, Loss: 2.2677810192108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10941 , TFLOPS: 99.96786226147637, Tokens per sec: 81686.3136555632, Loss: 2.2753634452819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10942 , TFLOPS: 98.87827276902021, Tokens per sec: 80795.98203274845, Loss: 2.2673330307006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10943 , TFLOPS: 100.61742852505533, Tokens per sec: 82217.09097085675, Loss: 2.2481296062469482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10944 , TFLOPS: 97.92298398984362, Tokens per sec: 80015.39097996238, Loss: 2.2335612773895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10945 , TFLOPS: 101.37254821933851, Tokens per sec: 82834.11871156577, Loss: 2.2481188774108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10946 , TFLOPS: 98.6881741885863, Tokens per sec: 80640.64758910312, Loss: 2.2657320499420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10947 , TFLOPS: 100.05388814628536, Tokens per sec: 81756.60762054389, Loss: 2.260938882827759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10948 , TFLOPS: 98.83114643894793, Tokens per sec: 80757.47389530673, Loss: 2.2273991107940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10949 , TFLOPS: 97.83620190592134, Tokens per sec: 79944.47910521991, Loss: 2.257688522338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10950 , TFLOPS: 100.66581430063955, Tokens per sec: 82256.6282336473, Loss: 2.2510643005371094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10951 , TFLOPS: 99.0998700486698, Tokens per sec: 80977.05487437159, Loss: 2.255892276763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10952 , TFLOPS: 98.50127406858343, Tokens per sec: 80487.92668981165, Loss: 2.2574081420898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10953 , TFLOPS: 99.82295595466152, Tokens per sec: 81567.90698204459, Loss: 2.27048659324646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10954 , TFLOPS: 100.24114322010668, Tokens per sec: 81909.61856173772, Loss: 2.239110231399536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10955 , TFLOPS: 99.40842216425617, Tokens per sec: 81229.18075085527, Loss: 2.2288777828216553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10956 , TFLOPS: 101.24079941100322, Tokens per sec: 82726.46336875895, Loss: 2.2500147819519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10957 , TFLOPS: 100.14494892303912, Tokens per sec: 81831.01572534232, Loss: 2.2684714794158936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10958 , TFLOPS: 100.22940054652197, Tokens per sec: 81900.02332086835, Loss: 2.265303611755371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10959 , TFLOPS: 100.82789196043572, Tokens per sec: 82389.06606171688, Loss: 2.257284641265869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10960 , TFLOPS: 100.10065669548558, Tokens per sec: 81794.82340602497, Loss: 2.266049861907959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10961 , TFLOPS: 98.74596757157, Tokens per sec: 80687.87205007311, Loss: 2.253964900970459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10962 , TFLOPS: 99.99591626772653, Tokens per sec: 81709.23730624455, Loss: 2.257291793823242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10963 , TFLOPS: 101.40749412258606, Tokens per sec: 82862.6739086969, Loss: 2.23711895942688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10964 , TFLOPS: 99.52754948252337, Tokens per sec: 81326.52274922139, Loss: 2.2710134983062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10965 , TFLOPS: 99.44923487145019, Tokens per sec: 81262.52986451607, Loss: 2.2700307369232178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10966 , TFLOPS: 99.98727132898519, Tokens per sec: 81702.17330425848, Loss: 2.2644293308258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10967 , TFLOPS: 100.62188487376324, Tokens per sec: 82220.7323681026, Loss: 2.2487716674804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10968 , TFLOPS: 99.4267758977007, Tokens per sec: 81244.17805891993, Loss: 2.278501510620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10969 , TFLOPS: 100.05676101575311, Tokens per sec: 81758.95511614023, Loss: 2.2722222805023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10970 , TFLOPS: 101.36980268976782, Tokens per sec: 82831.87526867734, Loss: 2.271455764770508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10971 , TFLOPS: 98.77212625218387, Tokens per sec: 80709.24698138704, Loss: 2.2668380737304688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10972 , TFLOPS: 101.32558611163523, Tokens per sec: 82795.74476444922, Loss: 2.268888235092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10973 , TFLOPS: 99.97405979162801, Tokens per sec: 81691.37781698871, Loss: 2.263254404067993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10974 , TFLOPS: 97.58887908490277, Tokens per sec: 79742.38526150983, Loss: 2.242483139038086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10975 , TFLOPS: 100.76547902565774, Tokens per sec: 82338.06684605792, Loss: 2.267333507537842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10976 , TFLOPS: 99.39754571159004, Tokens per sec: 81220.29332139698, Loss: 2.2513206005096436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10977 , TFLOPS: 99.47614716011952, Tokens per sec: 81284.52058837173, Loss: 2.257234573364258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10978 , TFLOPS: 98.9635715196388, Tokens per sec: 80865.6818376641, Loss: 2.267259359359741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10979 , TFLOPS: 100.59315717205756, Tokens per sec: 82197.25822352206, Loss: 2.260178327560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10980 , TFLOPS: 98.89167241138708, Tokens per sec: 80806.9312254639, Loss: 2.261409282684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10981 , TFLOPS: 99.24995139254835, Tokens per sec: 81099.69020389224, Loss: 2.2775604724884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10982 , TFLOPS: 98.37242942415045, Tokens per sec: 80382.64441408924, Loss: 2.257589340209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10983 , TFLOPS: 100.75435670860799, Tokens per sec: 82328.97851448265, Loss: 2.2369236946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10984 , TFLOPS: 98.27672578261314, Tokens per sec: 80304.4425049582, Loss: 2.2685494422912598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10985 , TFLOPS: 100.10299584064839, Tokens per sec: 81796.73478175244, Loss: 2.2283196449279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10986 , TFLOPS: 98.48596552108847, Tokens per sec: 80475.41768157644, Loss: 2.243191957473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10987 , TFLOPS: 98.29203206302249, Tokens per sec: 80316.94966069939, Loss: 2.270064353942871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10988 , TFLOPS: 100.71408942091355, Tokens per sec: 82296.07507713532, Loss: 2.2804229259490967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10989 , TFLOPS: 98.97223945948701, Tokens per sec: 80872.76463444669, Loss: 2.2687034606933594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10990 , TFLOPS: 99.25502572698139, Tokens per sec: 81103.83657318246, Loss: 2.276500940322876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10991 , TFLOPS: 100.08367239571989, Tokens per sec: 81780.94509747165, Loss: 2.2388999462127686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10992 , TFLOPS: 100.7473373614507, Tokens per sec: 82323.24282522636, Loss: 2.260874032974243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10993 , TFLOPS: 98.83053842415787, Tokens per sec: 80756.97707077008, Loss: 2.254329204559326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10994 , TFLOPS: 100.20228681610196, Tokens per sec: 81877.86799377264, Loss: 2.233203172683716 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10995 , TFLOPS: 99.96121766538508, Tokens per sec: 81680.88418504987, Loss: 2.264741897583008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10996 , TFLOPS: 100.37490256542274, Tokens per sec: 82018.91676606736, Loss: 2.262895107269287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10997 , TFLOPS: 100.04864178161834, Tokens per sec: 81752.32068092104, Loss: 2.258898973464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10998 , TFLOPS: 100.63566524772197, Tokens per sec: 82231.99266640267, Loss: 2.261324405670166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10999 , TFLOPS: 99.33871878576291, Tokens per sec: 81172.22432596386, Loss: 2.275421619415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11000 , TFLOPS: 101.36919414741972, Tokens per sec: 82831.37801305942, Loss: 2.2576520442962646 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/optimizer.pt +[rank0]:[2024-08-29 04:43:26,844] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007805035995261278, 'preprocessing_with_comm': 0.0017767830067896284, 'state_converting': 2.4063904380018357, : 2.417638369006454}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000, took 14.53s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009000 + eval ppl=8.015737533569336, eval loss=2.081406831741333 +------------------------------------------------------------------ +iteration: 11001 , TFLOPS: 97.87115521839446, Tokens per sec: 79973.04035662, Loss: 2.257368803024292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11002 , TFLOPS: 98.87071617338472, Tokens per sec: 80789.80734393062, Loss: 2.242385149002075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11003 , TFLOPS: 98.26132109643703, Tokens per sec: 80291.8549393318, Loss: 2.2401046752929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11004 , TFLOPS: 99.51561092062026, Tokens per sec: 81316.76744296444, Loss: 2.2557175159454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11005 , TFLOPS: 97.88666475177178, Tokens per sec: 79985.71359559351, Loss: 2.2726216316223145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11006 , TFLOPS: 100.60370262724047, Tokens per sec: 82205.87518641619, Loss: 2.261397123336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11007 , TFLOPS: 98.93779317123771, Tokens per sec: 80844.61768559183, Loss: 2.2452127933502197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11008 , TFLOPS: 99.433266532563, Tokens per sec: 81249.48172374973, Loss: 2.238276243209839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11009 , TFLOPS: 101.17270921244156, Tokens per sec: 82670.82511471657, Loss: 2.243351459503174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11010 , TFLOPS: 100.63441718715676, Tokens per sec: 82230.97284398689, Loss: 2.2441227436065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11011 , TFLOPS: 100.73259613007407, Tokens per sec: 82311.19738559546, Loss: 2.244990110397339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11012 , TFLOPS: 101.29180170992011, Tokens per sec: 82768.13866011992, Loss: 2.2513439655303955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11013 , TFLOPS: 100.44962152014023, Tokens per sec: 82079.97154740403, Loss: 2.264385938644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11014 , TFLOPS: 99.48647432072194, Tokens per sec: 81292.95917714472, Loss: 2.2665889263153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11015 , TFLOPS: 100.66590659721784, Tokens per sec: 82256.70365155733, Loss: 2.2574238777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11016 , TFLOPS: 100.59787952869678, Tokens per sec: 82201.11698269624, Loss: 2.269704580307007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11017 , TFLOPS: 99.84799094551126, Tokens per sec: 81588.3637175259, Loss: 2.263967990875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11018 , TFLOPS: 99.2131107277898, Tokens per sec: 81069.58674835495, Loss: 2.253207206726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11019 , TFLOPS: 101.32513656841562, Tokens per sec: 82795.3774311122, Loss: 2.252615451812744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11020 , TFLOPS: 100.0867597095987, Tokens per sec: 81783.46782111654, Loss: 2.2596938610076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11021 , TFLOPS: 100.1006066268687, Tokens per sec: 81794.78249366922, Loss: 2.2507572174072266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11022 , TFLOPS: 99.4258031546961, Tokens per sec: 81243.38320557024, Loss: 2.2553586959838867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11023 , TFLOPS: 99.96484627666499, Tokens per sec: 81683.84921673544, Loss: 2.2529778480529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11024 , TFLOPS: 99.62967493944416, Tokens per sec: 81409.97208901453, Loss: 2.2485761642456055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11025 , TFLOPS: 100.17378952932765, Tokens per sec: 81854.58212716393, Loss: 2.256983518600464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11026 , TFLOPS: 100.74151829291084, Tokens per sec: 82318.48791452628, Loss: 2.247265338897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11027 , TFLOPS: 98.26588106656901, Tokens per sec: 80295.58100831072, Loss: 2.2585971355438232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11028 , TFLOPS: 100.76846919367104, Tokens per sec: 82340.51018931535, Loss: 2.2705001831054688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11029 , TFLOPS: 100.60477382068906, Tokens per sec: 82206.750486158, Loss: 2.245267868041992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11030 , TFLOPS: 98.75585484343529, Tokens per sec: 80695.95119443526, Loss: 2.249500274658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11031 , TFLOPS: 100.60676310344124, Tokens per sec: 82208.37598030349, Loss: 2.268345594406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11032 , TFLOPS: 99.9009688815275, Tokens per sec: 81631.65335281818, Loss: 2.2397360801696777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11033 , TFLOPS: 99.77937737871449, Tokens per sec: 81532.29780582592, Loss: 2.2577173709869385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11034 , TFLOPS: 99.3060146794006, Tokens per sec: 81145.50095877661, Loss: 2.2444369792938232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11035 , TFLOPS: 98.82764043740943, Tokens per sec: 80754.60905119713, Loss: 2.265902519226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11036 , TFLOPS: 99.29572919730239, Tokens per sec: 81137.0964265823, Loss: 2.2388391494750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11037 , TFLOPS: 99.71361064792774, Tokens per sec: 81478.55811711379, Loss: 2.264296770095825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11038 , TFLOPS: 98.51737426733196, Tokens per sec: 80501.08257666505, Loss: 2.274930715560913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11039 , TFLOPS: 98.89442945332091, Tokens per sec: 80809.18407540061, Loss: 2.271118402481079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11040 , TFLOPS: 99.42069336188175, Tokens per sec: 81239.20786232388, Loss: 2.2476820945739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11041 , TFLOPS: 97.48818431985954, Tokens per sec: 79660.10497687914, Loss: 2.242802858352661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11042 , TFLOPS: 100.18603169057224, Tokens per sec: 81864.58551225816, Loss: 2.239790439605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11043 , TFLOPS: 98.96635925968417, Tokens per sec: 80867.95977181797, Loss: 2.272939682006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11044 , TFLOPS: 100.71035742806562, Tokens per sec: 82293.02556970924, Loss: 2.2693943977355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11045 , TFLOPS: 100.13988650329175, Tokens per sec: 81826.87909184847, Loss: 2.247509241104126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11046 , TFLOPS: 100.74329003164756, Tokens per sec: 82319.93564785668, Loss: 2.252969980239868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11047 , TFLOPS: 101.26478611086472, Tokens per sec: 82746.06351868837, Loss: 2.254368305206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11048 , TFLOPS: 101.27552915193074, Tokens per sec: 82754.8419340939, Loss: 2.249657392501831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11049 , TFLOPS: 100.11705152999563, Tokens per sec: 81808.2200473437, Loss: 2.2742247581481934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11050 , TFLOPS: 101.24483216043174, Tokens per sec: 82729.7586321292, Loss: 2.262075662612915 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11051 , TFLOPS: 100.48748298414137, Tokens per sec: 82110.90912428014, Loss: 2.254368782043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11052 , TFLOPS: 100.10003704150168, Tokens per sec: 81794.3170708034, Loss: 2.2651419639587402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11053 , TFLOPS: 100.73496106683918, Tokens per sec: 82313.12983630512, Loss: 2.251718282699585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11054 , TFLOPS: 99.76999409844622, Tokens per sec: 81524.63048597159, Loss: 2.2456555366516113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11055 , TFLOPS: 100.04926181560923, Tokens per sec: 81752.82732665609, Loss: 2.2443747520446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11056 , TFLOPS: 99.92201542571703, Tokens per sec: 81648.85102586169, Loss: 2.2705624103546143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11057 , TFLOPS: 101.26907555523105, Tokens per sec: 82749.56853410022, Loss: 2.2629687786102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11058 , TFLOPS: 100.03976014282411, Tokens per sec: 81745.06326522854, Loss: 2.2726824283599854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11059 , TFLOPS: 100.00981208133332, Tokens per sec: 81720.59193325274, Loss: 2.248958110809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11060 , TFLOPS: 99.29169648432355, Tokens per sec: 81133.80119299603, Loss: 2.241023302078247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11061 , TFLOPS: 100.599440356459, Tokens per sec: 82202.39237524003, Loss: 2.267592430114746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11062 , TFLOPS: 99.47876853049098, Tokens per sec: 81286.66257758232, Loss: 2.2602248191833496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11063 , TFLOPS: 100.30452328060312, Tokens per sec: 81961.40804071727, Loss: 2.2843174934387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11064 , TFLOPS: 99.92469834422238, Tokens per sec: 81651.04330763686, Loss: 2.2781012058258057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11065 , TFLOPS: 98.80351980926129, Tokens per sec: 80734.89946501706, Loss: 2.2682318687438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11066 , TFLOPS: 100.7576523415256, Tokens per sec: 82331.67146097677, Loss: 2.251924753189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11067 , TFLOPS: 100.57119535968774, Tokens per sec: 82179.31266128733, Loss: 2.2440500259399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11068 , TFLOPS: 98.69529676718517, Tokens per sec: 80646.46763142772, Loss: 2.250447988510132 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11069 , TFLOPS: 100.74710170166894, Tokens per sec: 82323.05026155226, Loss: 2.2685110569000244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11070 , TFLOPS: 99.92689727470412, Tokens per sec: 81652.84011033899, Loss: 2.254699945449829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11071 , TFLOPS: 99.23236478810732, Tokens per sec: 81085.319736684, Loss: 2.242403984069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11072 , TFLOPS: 99.46440391765476, Tokens per sec: 81274.9248826558, Loss: 2.266120195388794 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11073 , TFLOPS: 99.49952943782247, Tokens per sec: 81303.62684939617, Loss: 2.242560863494873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11074 , TFLOPS: 98.4156574204532, Tokens per sec: 80417.96712265584, Loss: 2.2706875801086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11075 , TFLOPS: 100.58433936353745, Tokens per sec: 82190.0529651897, Loss: 2.2611212730407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11076 , TFLOPS: 98.85349234567637, Tokens per sec: 80775.73331092935, Loss: 2.273632049560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11077 , TFLOPS: 99.03252836904925, Tokens per sec: 80922.02825442457, Loss: 2.2446658611297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11078 , TFLOPS: 99.58292134528969, Tokens per sec: 81371.76852368632, Loss: 2.2406206130981445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11079 , TFLOPS: 97.51102028581822, Tokens per sec: 79678.76483251406, Loss: 2.256545305252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11080 , TFLOPS: 99.83641707350446, Tokens per sec: 81578.90640877119, Loss: 2.2540459632873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11081 , TFLOPS: 98.96430761405657, Tokens per sec: 80866.2833193616, Loss: 2.2622792720794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11082 , TFLOPS: 100.67217054448341, Tokens per sec: 82261.82208411637, Loss: 2.253803253173828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11083 , TFLOPS: 99.37769305043794, Tokens per sec: 81204.07120091665, Loss: 2.281101942062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11084 , TFLOPS: 100.14021044737672, Tokens per sec: 81827.14379489936, Loss: 2.2533082962036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11085 , TFLOPS: 101.32883504675557, Tokens per sec: 82798.39955297107, Loss: 2.2354629039764404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11086 , TFLOPS: 100.55026987182467, Tokens per sec: 82162.21390648444, Loss: 2.25011944770813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11087 , TFLOPS: 100.10647589090007, Tokens per sec: 81799.57842039755, Loss: 2.254840850830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11088 , TFLOPS: 100.5404076204227, Tokens per sec: 82154.15520698695, Loss: 2.2696006298065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11089 , TFLOPS: 101.27373682277641, Tokens per sec: 82753.37737580323, Loss: 2.2649946212768555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11090 , TFLOPS: 99.92701083362387, Tokens per sec: 81652.93290225559, Loss: 2.2668471336364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11091 , TFLOPS: 100.67535682340957, Tokens per sec: 82264.42567464907, Loss: 2.2559070587158203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11092 , TFLOPS: 100.03033974957438, Tokens per sec: 81737.3656194017, Loss: 2.257132053375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11093 , TFLOPS: 99.90703709056025, Tokens per sec: 81636.61184262844, Loss: 2.2591114044189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11094 , TFLOPS: 100.63245116874911, Tokens per sec: 82229.36635973619, Loss: 2.262561798095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11095 , TFLOPS: 100.2685765751385, Tokens per sec: 81932.0350623316, Loss: 2.245680570602417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11096 , TFLOPS: 99.36931061698711, Tokens per sec: 81197.22169876093, Loss: 2.2455458641052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11097 , TFLOPS: 100.04560446357418, Tokens per sec: 81749.83881015962, Loss: 2.2555465698242188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11098 , TFLOPS: 100.0076548040409, Tokens per sec: 81718.82916644384, Loss: 2.252450942993164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11099 , TFLOPS: 100.46250723381111, Tokens per sec: 82090.50079575228, Loss: 2.2686405181884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11100 , TFLOPS: 99.36572920005287, Tokens per sec: 81194.2952307901, Loss: 2.2556614875793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11101 , TFLOPS: 97.52127148852428, Tokens per sec: 79687.1413541346, Loss: 2.2716012001037598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11102 , TFLOPS: 101.32211516553184, Tokens per sec: 82792.90856503803, Loss: 2.261385440826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11103 , TFLOPS: 98.85312814265366, Tokens per sec: 80775.43571126409, Loss: 2.2740514278411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11104 , TFLOPS: 100.07987482249398, Tokens per sec: 81777.84200263052, Loss: 2.270826816558838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11105 , TFLOPS: 101.31891400481207, Tokens per sec: 82790.2928142087, Loss: 2.241915464401245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11106 , TFLOPS: 96.93766152325264, Tokens per sec: 79210.25862805411, Loss: 2.2627127170562744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11107 , TFLOPS: 100.03289993683765, Tokens per sec: 81739.4576143196, Loss: 2.2435545921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11108 , TFLOPS: 100.67700644563637, Tokens per sec: 82265.77362343563, Loss: 2.2705702781677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11109 , TFLOPS: 99.39713781257808, Tokens per sec: 81219.96001661406, Loss: 2.2569093704223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11110 , TFLOPS: 99.45320698087816, Tokens per sec: 81265.77557738086, Loss: 2.2560622692108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11111 , TFLOPS: 98.36120002552299, Tokens per sec: 80373.4685834003, Loss: 2.2618343830108643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11112 , TFLOPS: 99.60772966546062, Tokens per sec: 81392.04004072095, Loss: 2.2697253227233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11113 , TFLOPS: 100.12034442786027, Tokens per sec: 81810.91075895564, Loss: 2.260819435119629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11114 , TFLOPS: 99.5519317261138, Tokens per sec: 81346.44610811393, Loss: 2.265413284301758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11115 , TFLOPS: 100.10766331170569, Tokens per sec: 81800.54869250476, Loss: 2.2574307918548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11116 , TFLOPS: 98.84288693871002, Tokens per sec: 80767.0673598897, Loss: 2.2845335006713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11117 , TFLOPS: 98.87059139542157, Tokens per sec: 80789.70538464485, Loss: 2.2574667930603027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11118 , TFLOPS: 100.02996697909828, Tokens per sec: 81737.0610190497, Loss: 2.2588019371032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11119 , TFLOPS: 99.55998901524909, Tokens per sec: 81353.02992647944, Loss: 2.2908248901367188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11120 , TFLOPS: 99.39609499951581, Tokens per sec: 81219.10790721647, Loss: 2.2727322578430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11121 , TFLOPS: 100.2277657350903, Tokens per sec: 81898.68747436376, Loss: 2.2718026638031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11122 , TFLOPS: 99.81044592126042, Tokens per sec: 81557.68471171563, Loss: 2.2585296630859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11123 , TFLOPS: 101.33799180069195, Tokens per sec: 82805.88177232831, Loss: 2.274240016937256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11124 , TFLOPS: 100.12358896173996, Tokens per sec: 81813.56195111045, Loss: 2.250371217727661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11125 , TFLOPS: 100.10770901206573, Tokens per sec: 81800.58603544536, Loss: 2.2630534172058105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11126 , TFLOPS: 100.64588073922168, Tokens per sec: 82240.34000747722, Loss: 2.272925853729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11127 , TFLOPS: 100.69410993161472, Tokens per sec: 82279.74932211143, Loss: 2.254774332046509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11128 , TFLOPS: 99.2613566477424, Tokens per sec: 81109.00973150869, Loss: 2.265345573425293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11129 , TFLOPS: 100.61044793813535, Tokens per sec: 82211.38695358814, Loss: 2.2609732151031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11130 , TFLOPS: 100.47964842842555, Tokens per sec: 82104.50730712534, Loss: 2.2673752307891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11131 , TFLOPS: 99.22544003608012, Tokens per sec: 81079.6613435436, Loss: 2.23925518989563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11132 , TFLOPS: 100.66557763992392, Tokens per sec: 82256.43485208435, Loss: 2.2370221614837646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11133 , TFLOPS: 100.59852341302272, Tokens per sec: 82201.64311715403, Loss: 2.2576816082000732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11134 , TFLOPS: 100.05391890262572, Tokens per sec: 81756.63275234133, Loss: 2.2964589595794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11135 , TFLOPS: 99.39435457749437, Tokens per sec: 81217.68576358032, Loss: 2.2714734077453613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11136 , TFLOPS: 100.1296955362521, Tokens per sec: 81818.55179034155, Loss: 2.267835855484009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11137 , TFLOPS: 99.79488327104377, Tokens per sec: 81544.96806960535, Loss: 2.249457836151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11138 , TFLOPS: 99.55393731063484, Tokens per sec: 81348.08492285441, Loss: 2.2697668075561523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11139 , TFLOPS: 99.45757322330844, Tokens per sec: 81269.34334646771, Loss: 2.2571041584014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11140 , TFLOPS: 101.37733917374729, Tokens per sec: 82838.03352374332, Loss: 2.2633190155029297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11141 , TFLOPS: 98.2266991935969, Tokens per sec: 80263.56449127407, Loss: 2.241661787033081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11142 , TFLOPS: 100.14509632009248, Tokens per sec: 81831.13616726897, Loss: 2.256889820098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11143 , TFLOPS: 100.52382453343911, Tokens per sec: 82140.60473972604, Loss: 2.240292549133301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11144 , TFLOPS: 98.8977250963176, Tokens per sec: 80811.8770301306, Loss: 2.254673719406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11145 , TFLOPS: 100.70564453472258, Tokens per sec: 82289.17454323797, Loss: 2.264688014984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11146 , TFLOPS: 100.76792713199721, Tokens per sec: 82340.06725676777, Loss: 2.283494234085083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11147 , TFLOPS: 99.22725611819055, Tokens per sec: 81081.14531098616, Loss: 2.2552616596221924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11148 , TFLOPS: 100.08303852875667, Tokens per sec: 81780.42714845881, Loss: 2.2762579917907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11149 , TFLOPS: 99.35983219424654, Tokens per sec: 81189.47663554331, Loss: 2.274951696395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11150 , TFLOPS: 100.1004184721847, Tokens per sec: 81794.62874763338, Loss: 2.2652227878570557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11151 , TFLOPS: 99.90024192842444, Tokens per sec: 81631.05934072421, Loss: 2.2525436878204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11152 , TFLOPS: 98.27184802928366, Tokens per sec: 80300.45676714815, Loss: 2.2660465240478516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11153 , TFLOPS: 100.16298434307278, Tokens per sec: 81845.75293132484, Loss: 2.253330707550049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11154 , TFLOPS: 98.64665331620591, Tokens per sec: 80606.71981543874, Loss: 2.269798517227173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11155 , TFLOPS: 99.02631337701364, Tokens per sec: 80916.94982444416, Loss: 2.2650980949401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11156 , TFLOPS: 100.19520282360209, Tokens per sec: 81872.0794811428, Loss: 2.27012300491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11157 , TFLOPS: 98.92079196065444, Tokens per sec: 80830.72555877415, Loss: 2.2573330402374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11158 , TFLOPS: 100.18902263285055, Tokens per sec: 81867.02948818747, Loss: 2.2364957332611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11159 , TFLOPS: 100.20585849942474, Tokens per sec: 81880.786508159, Loss: 2.2718677520751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11160 , TFLOPS: 99.25470732495636, Tokens per sec: 81103.57639869142, Loss: 2.2696430683135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11161 , TFLOPS: 101.2937782584768, Tokens per sec: 82769.75374882648, Loss: 2.2408909797668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11162 , TFLOPS: 99.93465069938772, Tokens per sec: 81659.17563323888, Loss: 2.2560648918151855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11163 , TFLOPS: 99.3985884997101, Tokens per sec: 81221.14541041365, Loss: 2.2491307258605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11164 , TFLOPS: 100.54670353402443, Tokens per sec: 82159.29976006216, Loss: 2.26602840423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11165 , TFLOPS: 100.7731506973626, Tokens per sec: 82344.33556649435, Loss: 2.261991500854492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11166 , TFLOPS: 98.89048531609012, Tokens per sec: 80805.9612193382, Loss: 2.2786059379577637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11167 , TFLOPS: 99.38576674111297, Tokens per sec: 81210.6684214025, Loss: 2.266289710998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11168 , TFLOPS: 100.80723078184857, Tokens per sec: 82372.18328082727, Loss: 2.266651153564453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11169 , TFLOPS: 98.18690593430696, Tokens per sec: 80231.04839473862, Loss: 2.269552707672119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11170 , TFLOPS: 100.69996616098392, Tokens per sec: 82284.53459788176, Loss: 2.250504493713379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11171 , TFLOPS: 100.21757016072426, Tokens per sec: 81890.35640809183, Loss: 2.2681350708007812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11172 , TFLOPS: 100.04501074999087, Tokens per sec: 81749.35367150715, Loss: 2.24292254447937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11173 , TFLOPS: 99.03025455896167, Tokens per sec: 80920.17026566894, Loss: 2.2514708042144775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11174 , TFLOPS: 99.44892109849924, Tokens per sec: 81262.27347256061, Loss: 2.2316272258758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11175 , TFLOPS: 100.63622336578669, Tokens per sec: 82232.44871904059, Loss: 2.27809476852417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11176 , TFLOPS: 99.00834189267106, Tokens per sec: 80902.26486196068, Loss: 2.2819559574127197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11177 , TFLOPS: 99.51046637890661, Tokens per sec: 81312.56370549795, Loss: 2.255237102508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11178 , TFLOPS: 100.56124808390656, Tokens per sec: 82171.18448618088, Loss: 2.255439519882202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11179 , TFLOPS: 98.57313318824889, Tokens per sec: 80546.6446263086, Loss: 2.235797643661499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11180 , TFLOPS: 100.13808221483471, Tokens per sec: 81825.4047613037, Loss: 2.2570667266845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11181 , TFLOPS: 101.33584896083693, Tokens per sec: 82804.13080272116, Loss: 2.2808420658111572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11182 , TFLOPS: 98.90569498165904, Tokens per sec: 80818.38942860618, Loss: 2.2500662803649902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11183 , TFLOPS: 100.66494959001024, Tokens per sec: 82255.92165633243, Loss: 2.2651541233062744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11184 , TFLOPS: 99.36646087197883, Tokens per sec: 81194.89309875571, Loss: 2.281238079071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11185 , TFLOPS: 99.94118443685815, Tokens per sec: 81664.51451831934, Loss: 2.253124475479126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11186 , TFLOPS: 99.94497939904782, Tokens per sec: 81667.61547961553, Loss: 2.2384490966796875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11187 , TFLOPS: 99.44949904839282, Tokens per sec: 81262.74573029642, Loss: 2.225787401199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11188 , TFLOPS: 99.52507334531192, Tokens per sec: 81324.49943376447, Loss: 2.2217626571655273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11189 , TFLOPS: 99.90839745733533, Tokens per sec: 81637.7234333393, Loss: 2.2444722652435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11190 , TFLOPS: 98.92404349149062, Tokens per sec: 80833.38246832223, Loss: 2.253891944885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11191 , TFLOPS: 100.27614494387419, Tokens per sec: 81938.2193712527, Loss: 2.249431610107422 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11192 , TFLOPS: 98.90634617376952, Tokens per sec: 80818.92153444284, Loss: 2.2708706855773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11193 , TFLOPS: 98.1731444882104, Tokens per sec: 80219.80356288199, Loss: 2.251232624053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11194 , TFLOPS: 99.43586550671661, Tokens per sec: 81251.60541243419, Loss: 2.2666070461273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11195 , TFLOPS: 99.46165231051596, Tokens per sec: 81272.67647363002, Loss: 2.265565872192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11196 , TFLOPS: 99.95226589302638, Tokens per sec: 81673.56946141645, Loss: 2.2600574493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11197 , TFLOPS: 99.48145389461065, Tokens per sec: 81288.85685773192, Loss: 2.2507545948028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11198 , TFLOPS: 100.12671186883094, Tokens per sec: 81816.11375888527, Loss: 2.276435375213623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11199 , TFLOPS: 100.17923080249929, Tokens per sec: 81859.0283315433, Loss: 2.264247417449951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11200 , TFLOPS: 101.2878507546701, Tokens per sec: 82764.9102328785, Loss: 2.2595276832580566 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/optimizer.pt +[rank0]:[2024-08-29 05:37:23,431] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0077438869993784465, 'preprocessing_with_comm': 0.0016546949991607107, 'state_converting': 2.3964790500031086, : 2.4075031350002973}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200, took 14.56s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009200 + eval ppl=7.669925212860107, eval loss=2.037306785583496 +------------------------------------------------------------------ +iteration: 11201 , TFLOPS: 97.08410124626349, Tokens per sec: 79329.91829542011, Loss: 2.2642760276794434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11202 , TFLOPS: 98.97687722038607, Tokens per sec: 80876.55426826389, Loss: 2.2611968517303467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11203 , TFLOPS: 99.38693408061523, Tokens per sec: 81211.6222845598, Loss: 2.2397685050964355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11204 , TFLOPS: 100.13442924487171, Tokens per sec: 81822.41982551113, Loss: 2.2568981647491455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11205 , TFLOPS: 98.59321078509697, Tokens per sec: 80563.05054753648, Loss: 2.2614595890045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11206 , TFLOPS: 100.367495084514, Tokens per sec: 82012.8639227313, Loss: 2.243595600128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11207 , TFLOPS: 100.67261624395914, Tokens per sec: 82262.18627663133, Loss: 2.263378381729126 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11208 , TFLOPS: 98.98494362873305, Tokens per sec: 80883.14553817203, Loss: 2.2665903568267822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11209 , TFLOPS: 100.61649076205596, Tokens per sec: 82216.32470056979, Loss: 2.280740261077881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11210 , TFLOPS: 101.30907930827175, Tokens per sec: 82782.25663049798, Loss: 2.2520415782928467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11211 , TFLOPS: 99.72636718353328, Tokens per sec: 81488.98181073953, Loss: 2.2520318031311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11212 , TFLOPS: 100.5267766486176, Tokens per sec: 82143.01698903258, Loss: 2.2605550289154053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11213 , TFLOPS: 100.767009492379, Tokens per sec: 82339.31742981357, Loss: 2.261934518814087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11214 , TFLOPS: 100.39273172076804, Tokens per sec: 82033.48542786328, Loss: 2.256063938140869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11215 , TFLOPS: 99.96271890694437, Tokens per sec: 81682.11088817337, Loss: 2.2747631072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11216 , TFLOPS: 99.4287704393188, Tokens per sec: 81245.80785022015, Loss: 2.2535789012908936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11217 , TFLOPS: 100.05129189047358, Tokens per sec: 81754.48615308652, Loss: 2.258512258529663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11218 , TFLOPS: 100.51515448955996, Tokens per sec: 82133.52022368558, Loss: 2.250293254852295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11219 , TFLOPS: 99.47332296022444, Tokens per sec: 81282.21286193607, Loss: 2.2639880180358887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11220 , TFLOPS: 100.58450377820708, Tokens per sec: 82190.18731264865, Loss: 2.264042377471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11221 , TFLOPS: 100.62227349514144, Tokens per sec: 82221.04992063473, Loss: 2.2689366340637207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11222 , TFLOPS: 98.8582031508368, Tokens per sec: 80779.58263109285, Loss: 2.2676820755004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11223 , TFLOPS: 99.9975103304504, Tokens per sec: 81710.53985593106, Loss: 2.2625691890716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11224 , TFLOPS: 100.12090989286938, Tokens per sec: 81811.37281497093, Loss: 2.2640604972839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11225 , TFLOPS: 100.60479554300392, Tokens per sec: 82206.76823602065, Loss: 2.2606446743011475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11226 , TFLOPS: 99.50180326737886, Tokens per sec: 81305.48485406028, Loss: 2.260852098464966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11227 , TFLOPS: 99.39149156462845, Tokens per sec: 81215.34632206707, Loss: 2.2757863998413086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11228 , TFLOPS: 100.32654547351706, Tokens per sec: 81979.40294144893, Loss: 2.2751102447509766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11229 , TFLOPS: 100.63724178130714, Tokens per sec: 82233.28089257868, Loss: 2.298259973526001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11230 , TFLOPS: 97.03569364821831, Tokens per sec: 79290.36320093497, Loss: 2.2458431720733643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11231 , TFLOPS: 101.31895834990559, Tokens per sec: 82790.32904972616, Loss: 2.2699928283691406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11232 , TFLOPS: 99.31150066365348, Tokens per sec: 81149.98369773144, Loss: 2.258571147918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11233 , TFLOPS: 98.69313528409167, Tokens per sec: 80644.7014279505, Loss: 2.251950979232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11234 , TFLOPS: 100.0532900635289, Tokens per sec: 81756.11891172756, Loss: 2.258333921432495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11235 , TFLOPS: 98.7123608034974, Tokens per sec: 80660.41109476582, Loss: 2.262373924255371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11236 , TFLOPS: 99.2873831172356, Tokens per sec: 81130.27662971252, Loss: 2.2751007080078125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11237 , TFLOPS: 97.87274213158243, Tokens per sec: 79974.33706423666, Loss: 2.267090082168579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11238 , TFLOPS: 100.76376381069964, Tokens per sec: 82336.6652997621, Loss: 2.250103712081909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11239 , TFLOPS: 98.05798627942819, Tokens per sec: 80125.70482605005, Loss: 2.24930739402771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11240 , TFLOPS: 98.59817789788012, Tokens per sec: 80567.10930325647, Loss: 2.242940664291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11241 , TFLOPS: 99.37225028053291, Tokens per sec: 81199.62377351789, Loss: 2.2625555992126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11242 , TFLOPS: 99.36531628607037, Tokens per sec: 81193.95782814565, Loss: 2.251047134399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11243 , TFLOPS: 98.6085957697235, Tokens per sec: 80575.62201451955, Loss: 2.253905773162842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11244 , TFLOPS: 99.68549612900033, Tokens per sec: 81455.5850199661, Loss: 2.256021499633789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11245 , TFLOPS: 100.79926887273037, Tokens per sec: 82365.677399929, Loss: 2.2502806186676025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11246 , TFLOPS: 99.05387659210332, Tokens per sec: 80939.47243703422, Loss: 2.252167224884033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11247 , TFLOPS: 100.57716567912746, Tokens per sec: 82184.19116299122, Loss: 2.2476534843444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11248 , TFLOPS: 101.0779999782304, Tokens per sec: 82593.43576141006, Loss: 2.2475643157958984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11249 , TFLOPS: 100.06633642664237, Tokens per sec: 81766.77943087084, Loss: 2.285050868988037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11250 , TFLOPS: 101.28263911769258, Tokens per sec: 82760.65167014471, Loss: 2.261200428009033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11251 , TFLOPS: 100.02965380586477, Tokens per sec: 81736.8051171388, Loss: 2.2498648166656494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11252 , TFLOPS: 100.67238843239586, Tokens per sec: 82262.00012593858, Loss: 2.2390451431274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11253 , TFLOPS: 99.91368708788926, Tokens per sec: 81642.04572663212, Loss: 2.2676424980163574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11254 , TFLOPS: 98.8140741035542, Tokens per sec: 80743.52365057552, Loss: 2.2701659202575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11255 , TFLOPS: 99.98425721129507, Tokens per sec: 81699.7103911036, Loss: 2.269178628921509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11256 , TFLOPS: 101.36517636551339, Tokens per sec: 82828.09498003685, Loss: 2.2713496685028076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11257 , TFLOPS: 99.4078347644102, Tokens per sec: 81228.70077132013, Loss: 2.2744545936584473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11258 , TFLOPS: 99.66184326885498, Tokens per sec: 81436.25765906257, Loss: 2.2668325901031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11259 , TFLOPS: 99.97796621931631, Tokens per sec: 81694.56985961321, Loss: 2.2612674236297607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11260 , TFLOPS: 99.46382946058527, Tokens per sec: 81274.45547899649, Loss: 2.260324716567993 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11261 , TFLOPS: 100.50308802710009, Tokens per sec: 82123.66040659125, Loss: 2.265545606613159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11262 , TFLOPS: 98.98932503386136, Tokens per sec: 80886.72569708979, Loss: 2.2768633365631104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11263 , TFLOPS: 101.30100464607571, Tokens per sec: 82775.65861615732, Loss: 2.2516653537750244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11264 , TFLOPS: 98.76006512207148, Tokens per sec: 80699.39152149111, Loss: 2.258932113647461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11265 , TFLOPS: 99.41149257867315, Tokens per sec: 81231.6896655149, Loss: 2.249807357788086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11266 , TFLOPS: 100.73159247371403, Tokens per sec: 82310.37727214726, Loss: 2.262023687362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11267 , TFLOPS: 99.51854865584893, Tokens per sec: 81319.16794204382, Loss: 2.2574868202209473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11268 , TFLOPS: 98.02179170617732, Tokens per sec: 80096.12930851555, Loss: 2.2525486946105957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11269 , TFLOPS: 99.56819257772035, Tokens per sec: 81359.7332685532, Loss: 2.2616922855377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11270 , TFLOPS: 100.79551503675756, Tokens per sec: 82362.6100439233, Loss: 2.251310110092163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11271 , TFLOPS: 97.42141215957265, Tokens per sec: 79605.54372584041, Loss: 2.2588305473327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11272 , TFLOPS: 100.67711237928344, Tokens per sec: 82265.86018454563, Loss: 2.25338077545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11273 , TFLOPS: 99.12066818401007, Tokens per sec: 80994.04956614976, Loss: 2.2371249198913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11274 , TFLOPS: 98.63760329945761, Tokens per sec: 80599.32481378554, Loss: 2.2564332485198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11275 , TFLOPS: 99.48560463316194, Tokens per sec: 81292.2485330517, Loss: 2.254850387573242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11276 , TFLOPS: 99.14620673204466, Tokens per sec: 81014.91777116954, Loss: 2.2446484565734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11277 , TFLOPS: 99.5215260622585, Tokens per sec: 81321.60085746634, Loss: 2.240060567855835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11278 , TFLOPS: 97.63160631479695, Tokens per sec: 79777.29878095312, Loss: 2.2665016651153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11279 , TFLOPS: 100.62475779464931, Tokens per sec: 82223.07990571417, Loss: 2.2544538974761963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11280 , TFLOPS: 99.4881017017208, Tokens per sec: 81294.2889520514, Loss: 2.267155170440674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11281 , TFLOPS: 99.68138450810905, Tokens per sec: 81452.22530869332, Loss: 2.2726733684539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11282 , TFLOPS: 100.61080532745574, Tokens per sec: 82211.67898560184, Loss: 2.2737109661102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11283 , TFLOPS: 100.12597329121775, Tokens per sec: 81815.51024810487, Loss: 2.257997989654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11284 , TFLOPS: 99.5613086535114, Tokens per sec: 81354.10823687409, Loss: 2.2750985622406006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11285 , TFLOPS: 100.59409803579167, Tokens per sec: 82198.02702749919, Loss: 2.269826650619507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11286 , TFLOPS: 99.37947322134015, Tokens per sec: 81205.52582438651, Loss: 2.2731332778930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11287 , TFLOPS: 99.98620006990645, Tokens per sec: 81701.29795088859, Loss: 2.247098684310913 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11288 , TFLOPS: 101.31565997769222, Tokens per sec: 82787.63386488297, Loss: 2.2522830963134766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11289 , TFLOPS: 100.07007130274292, Tokens per sec: 81769.83129427688, Loss: 2.2770118713378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11290 , TFLOPS: 100.09040390510889, Tokens per sec: 81786.44558707786, Loss: 2.249370574951172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11291 , TFLOPS: 100.6297030356148, Tokens per sec: 82227.1207894089, Loss: 2.2506790161132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11292 , TFLOPS: 98.78984425665722, Tokens per sec: 80723.72481894423, Loss: 2.2379844188690186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11293 , TFLOPS: 100.19802979624731, Tokens per sec: 81874.38947326396, Loss: 2.276233196258545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11294 , TFLOPS: 100.63336027900912, Tokens per sec: 82230.10921713224, Loss: 2.246187686920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11295 , TFLOPS: 99.40719064571637, Tokens per sec: 81228.1744453543, Loss: 2.2487006187438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11296 , TFLOPS: 98.40977034928446, Tokens per sec: 80413.15664525717, Loss: 2.2701470851898193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11297 , TFLOPS: 100.78530360802257, Tokens per sec: 82354.26602263837, Loss: 2.2414662837982178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11298 , TFLOPS: 99.71082686958478, Tokens per sec: 81476.28342016894, Loss: 2.2634596824645996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11299 , TFLOPS: 100.63253087818424, Tokens per sec: 82229.43149236753, Loss: 2.2521092891693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11300 , TFLOPS: 98.42510803367027, Tokens per sec: 80425.68946199593, Loss: 2.2597503662109375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11301 , TFLOPS: 99.50614382608455, Tokens per sec: 81309.03163631463, Loss: 2.250502586364746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11302 , TFLOPS: 100.67396469646268, Tokens per sec: 82263.28813188421, Loss: 2.2687878608703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11303 , TFLOPS: 98.33634724334951, Tokens per sec: 80353.16073531883, Loss: 2.283336877822876 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11304 , TFLOPS: 101.30991277768112, Tokens per sec: 82782.9376798078, Loss: 2.2466564178466797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11305 , TFLOPS: 98.69201472287757, Tokens per sec: 80643.78578853665, Loss: 2.2775237560272217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11306 , TFLOPS: 98.74877157912412, Tokens per sec: 80690.16327682717, Loss: 2.2438406944274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11307 , TFLOPS: 100.75830190210809, Tokens per sec: 82332.20223364989, Loss: 2.258565664291382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11308 , TFLOPS: 99.6018152889741, Tokens per sec: 81387.20725144423, Loss: 2.2791640758514404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11309 , TFLOPS: 98.83519759828268, Tokens per sec: 80760.78420188528, Loss: 2.2586512565612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11310 , TFLOPS: 100.04436622071398, Tokens per sec: 81748.82701004334, Loss: 2.2878007888793945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11311 , TFLOPS: 98.25867530077566, Tokens per sec: 80289.69299158808, Loss: 2.2501437664031982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11312 , TFLOPS: 100.59094531989719, Tokens per sec: 82195.45086218367, Loss: 2.2497975826263428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11313 , TFLOPS: 98.45781651361719, Tokens per sec: 80452.41640295174, Loss: 2.235741376876831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11314 , TFLOPS: 100.7792852184638, Tokens per sec: 82349.34824160275, Loss: 2.2623484134674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11315 , TFLOPS: 98.2416994233735, Tokens per sec: 80275.82156516479, Loss: 2.2419188022613525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11316 , TFLOPS: 98.55280459119938, Tokens per sec: 80530.0336063548, Loss: 2.264723062515259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11317 , TFLOPS: 101.37935274459485, Tokens per sec: 82839.67886431678, Loss: 2.23508620262146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11318 , TFLOPS: 98.30725478868452, Tokens per sec: 80329.38854170566, Loss: 2.2807672023773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11319 , TFLOPS: 100.01790517057272, Tokens per sec: 81727.20500480488, Loss: 2.2706894874572754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11320 , TFLOPS: 100.71022038332235, Tokens per sec: 82292.91358692167, Loss: 2.2333688735961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11321 , TFLOPS: 100.64732962748614, Tokens per sec: 82241.52393137582, Loss: 2.25659441947937 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11322 , TFLOPS: 99.87689345096847, Tokens per sec: 81611.98069875166, Loss: 2.272373676300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11323 , TFLOPS: 100.715722497167, Tokens per sec: 82297.40950578114, Loss: 2.264275074005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11324 , TFLOPS: 98.79267406807665, Tokens per sec: 80726.03713070088, Loss: 2.2476139068603516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11325 , TFLOPS: 98.9680724027207, Tokens per sec: 80869.35962509335, Loss: 2.2690579891204834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11326 , TFLOPS: 100.79368111039294, Tokens per sec: 82361.1114954813, Loss: 2.252182960510254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11327 , TFLOPS: 100.1873140271409, Tokens per sec: 81865.63334247869, Loss: 2.2752113342285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11328 , TFLOPS: 98.22826295956249, Tokens per sec: 80264.84228469947, Loss: 2.2378242015838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11329 , TFLOPS: 100.63753938049427, Tokens per sec: 82233.52406853538, Loss: 2.2602298259735107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11330 , TFLOPS: 99.99318848602326, Tokens per sec: 81707.00836559547, Loss: 2.256927013397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11331 , TFLOPS: 99.46142694608913, Tokens per sec: 81272.49232255547, Loss: 2.277559280395508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11332 , TFLOPS: 100.16926798915146, Tokens per sec: 81850.88746029117, Loss: 2.242920160293579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11333 , TFLOPS: 99.66971089026488, Tokens per sec: 81442.6864950472, Loss: 2.2446725368499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11334 , TFLOPS: 99.64874320074254, Tokens per sec: 81425.55325618213, Loss: 2.2461979389190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11335 , TFLOPS: 99.41336547049855, Tokens per sec: 81233.22005363795, Loss: 2.2817769050598145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11336 , TFLOPS: 99.81524168085473, Tokens per sec: 81561.60345033398, Loss: 2.2547383308410645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11337 , TFLOPS: 99.28938867247909, Tokens per sec: 81131.91542052956, Loss: 2.2799768447875977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11338 , TFLOPS: 100.16457108963527, Tokens per sec: 81847.04950278754, Loss: 2.2408719062805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11339 , TFLOPS: 99.54169298714926, Tokens per sec: 81338.07977093742, Loss: 2.2739319801330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11340 , TFLOPS: 101.31249319540217, Tokens per sec: 82785.04620555353, Loss: 2.2366843223571777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11341 , TFLOPS: 97.48192678186639, Tokens per sec: 79654.9917815016, Loss: 2.2664105892181396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11342 , TFLOPS: 101.22309757715668, Tokens per sec: 82711.9987446372, Loss: 2.2559726238250732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11343 , TFLOPS: 97.5375939350934, Tokens per sec: 79700.47884540338, Loss: 2.248335123062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11344 , TFLOPS: 97.8425507178051, Tokens per sec: 79949.66688284308, Loss: 2.2765703201293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11345 , TFLOPS: 99.8827008998133, Tokens per sec: 81616.7261146998, Loss: 2.245980978012085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11346 , TFLOPS: 99.1437791630701, Tokens per sec: 81012.93414206924, Loss: 2.2581639289855957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11347 , TFLOPS: 99.57203776107283, Tokens per sec: 81362.87526685433, Loss: 2.254567861557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11348 , TFLOPS: 97.16039908528411, Tokens per sec: 79392.26322376498, Loss: 2.2699668407440186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11349 , TFLOPS: 100.09128598348677, Tokens per sec: 81787.16635602714, Loss: 2.2685813903808594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11350 , TFLOPS: 100.03475686516651, Tokens per sec: 81740.9749582587, Loss: 2.24566650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11351 , TFLOPS: 99.50915469922481, Tokens per sec: 81311.49189826428, Loss: 2.2650623321533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11352 , TFLOPS: 101.34167428646263, Tokens per sec: 82808.89082624715, Loss: 2.261629581451416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11353 , TFLOPS: 98.91211331154977, Tokens per sec: 80823.63401118241, Loss: 2.2580442428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11354 , TFLOPS: 99.1825328655603, Tokens per sec: 81044.60078998304, Loss: 2.2500672340393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11355 , TFLOPS: 100.62813534678035, Tokens per sec: 82225.83979050638, Loss: 2.2688870429992676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11356 , TFLOPS: 98.29542866959058, Tokens per sec: 80319.72511536261, Loss: 2.249595880508423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11357 , TFLOPS: 98.92015589933938, Tokens per sec: 80830.20581669993, Loss: 2.2521989345550537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11358 , TFLOPS: 100.67993533023956, Tokens per sec: 82268.16689044109, Loss: 2.2558798789978027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11359 , TFLOPS: 99.97612333854015, Tokens per sec: 81693.06399429103, Loss: 2.254828929901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11360 , TFLOPS: 100.0413304338607, Tokens per sec: 81746.34639045743, Loss: 2.239403009414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11361 , TFLOPS: 99.39658812341244, Tokens per sec: 81219.51085144655, Loss: 2.2566723823547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11362 , TFLOPS: 99.40793358810133, Tokens per sec: 81228.78152270209, Loss: 2.2483153343200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11363 , TFLOPS: 100.0121803983086, Tokens per sec: 81722.52714601914, Loss: 2.258023977279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11364 , TFLOPS: 99.57257922234945, Tokens per sec: 81363.31770880191, Loss: 2.263551712036133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11365 , TFLOPS: 100.63275536207905, Tokens per sec: 82229.61492393671, Loss: 2.254357099533081 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11366 , TFLOPS: 98.38639355141214, Tokens per sec: 80394.05486194338, Loss: 2.2686336040496826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11367 , TFLOPS: 100.21449589430254, Tokens per sec: 81887.84434585998, Loss: 2.26149845123291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11368 , TFLOPS: 100.67482426581392, Tokens per sec: 82263.9905081284, Loss: 2.2447352409362793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11369 , TFLOPS: 99.42128361015587, Tokens per sec: 81239.690169383, Loss: 2.260080099105835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11370 , TFLOPS: 101.23369564991717, Tokens per sec: 82720.65870270845, Loss: 2.256168842315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11371 , TFLOPS: 99.22776678057173, Tokens per sec: 81081.56258636343, Loss: 2.243561267852783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11372 , TFLOPS: 100.10762839673106, Tokens per sec: 81800.5201625802, Loss: 2.2449378967285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11373 , TFLOPS: 100.06508775493064, Tokens per sec: 81765.75910907153, Loss: 2.248281955718994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11374 , TFLOPS: 101.2088571610586, Tokens per sec: 82700.36253406254, Loss: 2.2624917030334473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11375 , TFLOPS: 98.8840216704913, Tokens per sec: 80800.67960813036, Loss: 2.254157781600952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11376 , TFLOPS: 99.45244842341577, Tokens per sec: 81265.15574055133, Loss: 2.256579875946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11377 , TFLOPS: 98.32474885879077, Tokens per sec: 80343.6833967269, Loss: 2.2624497413635254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11378 , TFLOPS: 100.77347438118375, Tokens per sec: 82344.60005687698, Loss: 2.25644850730896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11379 , TFLOPS: 98.41460552395944, Tokens per sec: 80417.10759095257, Loss: 2.2501516342163086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11380 , TFLOPS: 100.6465587825249, Tokens per sec: 82240.89405411467, Loss: 2.2607803344726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11381 , TFLOPS: 98.82623961005687, Tokens per sec: 80753.46439910684, Loss: 2.261385917663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11382 , TFLOPS: 98.8281864735108, Tokens per sec: 80755.05523135177, Loss: 2.278270721435547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11383 , TFLOPS: 100.8300806354369, Tokens per sec: 82390.85448440185, Loss: 2.253445625305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11384 , TFLOPS: 99.71521290731098, Tokens per sec: 81479.86736450168, Loss: 2.2543325424194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11385 , TFLOPS: 99.28589174644999, Tokens per sec: 81129.05799225229, Loss: 2.2704482078552246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11386 , TFLOPS: 98.9743293637906, Tokens per sec: 80874.47234905207, Loss: 2.2281763553619385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11387 , TFLOPS: 100.19676341714545, Tokens per sec: 81873.35468230034, Loss: 2.2533555030822754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11388 , TFLOPS: 99.90373241790361, Tokens per sec: 81633.91150953085, Loss: 2.268040657043457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11389 , TFLOPS: 98.73175519688266, Tokens per sec: 80676.25875285701, Loss: 2.249985694885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11390 , TFLOPS: 100.59915363454549, Tokens per sec: 82202.15808738364, Loss: 2.2445414066314697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11391 , TFLOPS: 100.13153740146885, Tokens per sec: 81820.05682582397, Loss: 2.2658164501190186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11392 , TFLOPS: 97.60856649313814, Tokens per sec: 79758.47234957792, Loss: 2.288712501525879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11393 , TFLOPS: 100.78292191526717, Tokens per sec: 82352.3198801777, Loss: 2.295362949371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11394 , TFLOPS: 99.02420832969656, Tokens per sec: 80915.22973609263, Loss: 2.2577121257781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11395 , TFLOPS: 99.4041590770409, Tokens per sec: 81225.69727255005, Loss: 2.266752004623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11396 , TFLOPS: 100.6325816884472, Tokens per sec: 82229.47301074138, Loss: 2.24698543548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11397 , TFLOPS: 99.45854265127367, Tokens per sec: 81270.13549101341, Loss: 2.26318359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11398 , TFLOPS: 99.27110806062716, Tokens per sec: 81116.97786200052, Loss: 2.2501072883605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11399 , TFLOPS: 99.4029931974789, Tokens per sec: 81224.74460234755, Loss: 2.2463598251342773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11400 , TFLOPS: 100.71441870163744, Tokens per sec: 82296.34414089126, Loss: 2.2464351654052734 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/model.pt +[rank0]:[2024-08-29 06:31:24,858] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007670221995795146, 'preprocessing_with_comm': 0.001798980993044097, 'state_converting': 2.4369989889964927, : 2.4480849679966923}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400, took 14.70s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009400 + eval ppl=7.839338779449463, eval loss=2.059154510498047 +------------------------------------------------------------------ +iteration: 11401 , TFLOPS: 99.21097590171549, Tokens per sec: 81067.84232701432, Loss: 2.2733681201934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11402 , TFLOPS: 98.86817676846532, Tokens per sec: 80787.73233080088, Loss: 2.2212159633636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11403 , TFLOPS: 100.70282268712744, Tokens per sec: 82286.86873892717, Loss: 2.246063470840454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11404 , TFLOPS: 99.41338050772266, Tokens per sec: 81233.23234094087, Loss: 2.252898693084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11405 , TFLOPS: 98.83595343696241, Tokens per sec: 80761.40181712751, Loss: 2.2522027492523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11406 , TFLOPS: 100.12074896317156, Tokens per sec: 81811.24131517216, Loss: 2.2708311080932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11407 , TFLOPS: 99.59280811706107, Tokens per sec: 81379.84725940885, Loss: 2.260206699371338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11408 , TFLOPS: 99.66916229647113, Tokens per sec: 81442.23822493626, Loss: 2.259793519973755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11409 , TFLOPS: 100.0872249094069, Tokens per sec: 81783.84794785513, Loss: 2.258549213409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11410 , TFLOPS: 100.66779851894754, Tokens per sec: 82258.24958950507, Loss: 2.277578115463257 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11411 , TFLOPS: 100.58474448759496, Tokens per sec: 82190.3840024859, Loss: 2.262148141860962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11412 , TFLOPS: 99.31610575279812, Tokens per sec: 81153.74663461707, Loss: 2.2618048191070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11413 , TFLOPS: 101.13968762259636, Tokens per sec: 82643.84232360272, Loss: 2.2531750202178955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11414 , TFLOPS: 100.73280141264465, Tokens per sec: 82311.36512726833, Loss: 2.258594512939453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11415 , TFLOPS: 98.8045649999309, Tokens per sec: 80735.7535172188, Loss: 2.272148847579956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11416 , TFLOPS: 99.93774288404732, Tokens per sec: 81661.70233692417, Loss: 2.2306032180786133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11417 , TFLOPS: 99.47355431320194, Tokens per sec: 81282.40190640955, Loss: 2.239515781402588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11418 , TFLOPS: 100.51671397207625, Tokens per sec: 82134.7945169943, Loss: 2.2610580921173096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11419 , TFLOPS: 100.52669198965769, Tokens per sec: 82142.94781201711, Loss: 2.259958028793335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11420 , TFLOPS: 101.26685892221978, Tokens per sec: 82747.7572662055, Loss: 2.2739081382751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11421 , TFLOPS: 98.27917530495321, Tokens per sec: 80306.44407271859, Loss: 2.2655160427093506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11422 , TFLOPS: 100.70218083962457, Tokens per sec: 82286.34426880995, Loss: 2.2556312084198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11423 , TFLOPS: 99.42677531278223, Tokens per sec: 81244.177580968, Loss: 2.252667188644409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11424 , TFLOPS: 100.58583280746075, Tokens per sec: 82191.27329666408, Loss: 2.257549285888672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11425 , TFLOPS: 98.70362426200242, Tokens per sec: 80653.27224181168, Loss: 2.2542786598205566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11426 , TFLOPS: 100.10323976795635, Tokens per sec: 81796.93410103503, Loss: 2.2486443519592285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11427 , TFLOPS: 99.43886505842505, Tokens per sec: 81254.05642335048, Loss: 2.2451112270355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11428 , TFLOPS: 100.68353891166645, Tokens per sec: 82271.11146958944, Loss: 2.242079973220825 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11429 , TFLOPS: 100.15546899371333, Tokens per sec: 81839.61194590094, Loss: 2.269012689590454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11430 , TFLOPS: 98.73391950730415, Tokens per sec: 80678.02726661673, Loss: 2.2738304138183594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11431 , TFLOPS: 100.78777369476462, Tokens per sec: 82356.28439410099, Loss: 2.273967742919922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11432 , TFLOPS: 99.9680122124668, Tokens per sec: 81686.43618437756, Loss: 2.245298147201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11433 , TFLOPS: 99.4204507242871, Tokens per sec: 81239.00959689922, Loss: 2.28053879737854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11434 , TFLOPS: 99.94088610834753, Tokens per sec: 81664.27074641365, Loss: 2.2504067420959473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11435 , TFLOPS: 99.94092409838264, Tokens per sec: 81664.30178904929, Loss: 2.260446786880493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11436 , TFLOPS: 98.79358674992127, Tokens per sec: 80726.78290653064, Loss: 2.2701878547668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11437 , TFLOPS: 99.96302396890029, Tokens per sec: 81682.36016215054, Loss: 2.229372501373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11438 , TFLOPS: 99.47621418293053, Tokens per sec: 81284.57535443592, Loss: 2.28495717048645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11439 , TFLOPS: 100.0529529215384, Tokens per sec: 81755.84342432833, Loss: 2.2577366828918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11440 , TFLOPS: 98.06633705221319, Tokens per sec: 80132.52845746066, Loss: 2.256312370300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11441 , TFLOPS: 100.70988691671359, Tokens per sec: 82292.64110277145, Loss: 2.2482786178588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11442 , TFLOPS: 98.80385460270872, Tokens per sec: 80735.17303336131, Loss: 2.279893398284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11443 , TFLOPS: 99.62895253511154, Tokens per sec: 81409.38179383798, Loss: 2.252469301223755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11444 , TFLOPS: 100.2649509941121, Tokens per sec: 81929.07250674418, Loss: 2.2440872192382812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11445 , TFLOPS: 99.53649611306835, Tokens per sec: 81333.83327134646, Loss: 2.275399684906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11446 , TFLOPS: 99.05929904992132, Tokens per sec: 80943.9032669038, Loss: 2.2832398414611816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11447 , TFLOPS: 100.72179891289498, Tokens per sec: 82302.37470149333, Loss: 2.253981828689575 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11448 , TFLOPS: 101.36534459706391, Tokens per sec: 82828.23244636746, Loss: 2.232802152633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11449 , TFLOPS: 100.09481094554245, Tokens per sec: 81790.04669326343, Loss: 2.2399299144744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11450 , TFLOPS: 99.9945232700002, Tokens per sec: 81708.09905194343, Loss: 2.266648292541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11451 , TFLOPS: 101.36513107126876, Tokens per sec: 82828.0579689436, Loss: 2.2478952407836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11452 , TFLOPS: 101.27352076358255, Tokens per sec: 82753.20082827409, Loss: 2.2709832191467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11453 , TFLOPS: 98.87940289808263, Tokens per sec: 80796.90549029746, Loss: 2.243177652359009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11454 , TFLOPS: 99.93305633262055, Tokens per sec: 81657.87283511077, Loss: 2.280001163482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11455 , TFLOPS: 100.60196634691893, Tokens per sec: 82204.45642706963, Loss: 2.2661962509155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11456 , TFLOPS: 99.78955515795595, Tokens per sec: 81540.61433124346, Loss: 2.2478132247924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11457 , TFLOPS: 100.66889622980254, Tokens per sec: 82259.14655730229, Loss: 2.244706630706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11458 , TFLOPS: 100.74145111550348, Tokens per sec: 82318.43302213743, Loss: 2.2416398525238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11459 , TFLOPS: 99.23414535563701, Tokens per sec: 81086.7746842484, Loss: 2.253040075302124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11460 , TFLOPS: 100.7138860212882, Tokens per sec: 82295.90887406548, Loss: 2.257650852203369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11461 , TFLOPS: 99.30871908978997, Tokens per sec: 81147.71080212349, Loss: 2.2626731395721436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11462 , TFLOPS: 100.68678090453078, Tokens per sec: 82273.7605854152, Loss: 2.2611887454986572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11463 , TFLOPS: 98.80859718575651, Tokens per sec: 80739.04832005462, Loss: 2.270620346069336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11464 , TFLOPS: 99.98280178704212, Tokens per sec: 81698.52112648055, Loss: 2.2569198608398438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11465 , TFLOPS: 99.35222511715018, Tokens per sec: 81183.2606970243, Loss: 2.2782771587371826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11466 , TFLOPS: 100.26328210899118, Tokens per sec: 81927.70881775094, Loss: 2.257073163986206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11467 , TFLOPS: 100.1981321043953, Tokens per sec: 81874.47307188534, Loss: 2.2609827518463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11468 , TFLOPS: 98.2386268340822, Tokens per sec: 80273.31087335938, Loss: 2.2577991485595703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11469 , TFLOPS: 100.79086457677883, Tokens per sec: 82358.81003336118, Loss: 2.2557055950164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11470 , TFLOPS: 100.07549129636783, Tokens per sec: 81774.26011059084, Loss: 2.256751298904419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11471 , TFLOPS: 99.40795132554926, Tokens per sec: 81228.7960164274, Loss: 2.270679473876953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11472 , TFLOPS: 99.90586930014764, Tokens per sec: 81635.65761102068, Loss: 2.229748010635376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11473 , TFLOPS: 100.65642995765886, Tokens per sec: 82248.96004542329, Loss: 2.228996753692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11474 , TFLOPS: 98.93593230298094, Tokens per sec: 80843.09712223623, Loss: 2.258633852005005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11475 , TFLOPS: 100.08030215541031, Tokens per sec: 81778.19118735698, Loss: 2.2492988109588623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11476 , TFLOPS: 98.77751827018227, Tokens per sec: 80713.65293809643, Loss: 2.271573066711426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11477 , TFLOPS: 100.07326031108406, Tokens per sec: 81772.4371150848, Loss: 2.2481281757354736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11478 , TFLOPS: 98.19165234481193, Tokens per sec: 80234.92680894572, Loss: 2.257045269012451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11479 , TFLOPS: 100.66466299944285, Tokens per sec: 82255.68747580233, Loss: 2.2522666454315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11480 , TFLOPS: 99.46280405216176, Tokens per sec: 81273.61759137727, Loss: 2.24524188041687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11481 , TFLOPS: 98.9104309865787, Tokens per sec: 80822.25934014167, Loss: 2.2544121742248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11482 , TFLOPS: 100.16783949984287, Tokens per sec: 81849.72020490453, Loss: 2.2530770301818848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11483 , TFLOPS: 99.49874613923065, Tokens per sec: 81302.9867959529, Loss: 2.237639904022217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11484 , TFLOPS: 99.69142834604523, Tokens per sec: 81460.43238722236, Loss: 2.2673494815826416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11485 , TFLOPS: 100.13909462012765, Tokens per sec: 81826.23202373019, Loss: 2.2676892280578613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11486 , TFLOPS: 101.24289240301034, Tokens per sec: 82728.17360640639, Loss: 2.257995128631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11487 , TFLOPS: 99.97782523409694, Tokens per sec: 81694.45465696118, Loss: 2.256436824798584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11488 , TFLOPS: 100.06028981950318, Tokens per sec: 81761.83859252391, Loss: 2.250969409942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11489 , TFLOPS: 101.31428509388624, Tokens per sec: 82786.51041193244, Loss: 2.257375955581665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11490 , TFLOPS: 101.25813117218047, Tokens per sec: 82740.6255969761, Loss: 2.2566564083099365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11491 , TFLOPS: 99.2603702885155, Tokens per sec: 81108.20375199326, Loss: 2.2427077293395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11492 , TFLOPS: 99.83051805821579, Tokens per sec: 81574.08617152464, Loss: 2.2537643909454346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11493 , TFLOPS: 99.94451255926421, Tokens per sec: 81667.23401281082, Loss: 2.250473737716675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11494 , TFLOPS: 100.51547869532916, Tokens per sec: 82133.78514056541, Loss: 2.2529032230377197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11495 , TFLOPS: 99.97474306201605, Tokens per sec: 81691.93613481171, Loss: 2.274324655532837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11496 , TFLOPS: 101.30110429779198, Tokens per sec: 82775.74004413994, Loss: 2.2467644214630127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11497 , TFLOPS: 98.61729722192659, Tokens per sec: 80582.73219511002, Loss: 2.2549314498901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11498 , TFLOPS: 100.64669322513447, Tokens per sec: 82241.00391063181, Loss: 2.267322540283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11499 , TFLOPS: 98.86049816856682, Tokens per sec: 80781.45794915907, Loss: 2.250962257385254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11500 , TFLOPS: 100.69064795185263, Tokens per sec: 82276.92044932855, Loss: 2.266010284423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11501 , TFLOPS: 99.38360107786507, Tokens per sec: 81208.8988022135, Loss: 2.2535111904144287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11502 , TFLOPS: 100.64070227210358, Tokens per sec: 82236.10854868942, Loss: 2.261383295059204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11503 , TFLOPS: 98.13391207109179, Tokens per sec: 80187.74574492211, Loss: 2.260911464691162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11504 , TFLOPS: 100.68028625008212, Tokens per sec: 82268.45363607792, Loss: 2.241095542907715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11505 , TFLOPS: 100.68529862323271, Tokens per sec: 82272.54937520924, Loss: 2.257478713989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11506 , TFLOPS: 98.27396865050515, Tokens per sec: 80302.18958133788, Loss: 2.2603371143341064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11507 , TFLOPS: 100.80077109795984, Tokens per sec: 82366.90490683472, Loss: 2.2639806270599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11508 , TFLOPS: 99.3592522425638, Tokens per sec: 81189.00274209406, Loss: 2.247445583343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11509 , TFLOPS: 99.8978659011367, Tokens per sec: 81629.11782766302, Loss: 2.2751624584198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11510 , TFLOPS: 99.49697443088134, Tokens per sec: 81301.53908745284, Loss: 2.2613110542297363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11511 , TFLOPS: 100.64636402746075, Tokens per sec: 82240.73491473832, Loss: 2.2514796257019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11512 , TFLOPS: 98.13784665353224, Tokens per sec: 80190.96079351936, Loss: 2.243716239929199 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11513 , TFLOPS: 100.1219683448683, Tokens per sec: 81812.23770334624, Loss: 2.250856876373291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11514 , TFLOPS: 99.58672224269307, Tokens per sec: 81374.87433479843, Loss: 2.264723300933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11515 , TFLOPS: 100.11550849403068, Tokens per sec: 81806.95919293545, Loss: 2.2665326595306396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11516 , TFLOPS: 98.03752489806304, Tokens per sec: 80108.98530461376, Loss: 2.266199827194214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11517 , TFLOPS: 100.13647236993212, Tokens per sec: 81824.0893155927, Loss: 2.2469534873962402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11518 , TFLOPS: 99.34419571576389, Tokens per sec: 81176.69966645638, Loss: 2.255906820297241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11519 , TFLOPS: 99.10103568520644, Tokens per sec: 80978.00734599183, Loss: 2.252998113632202 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11520 , TFLOPS: 100.84904549768453, Tokens per sec: 82406.35116154337, Loss: 2.278775930404663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11521 , TFLOPS: 99.58718780899842, Tokens per sec: 81375.25476101127, Loss: 2.259998321533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11522 , TFLOPS: 100.15618836411215, Tokens per sec: 81840.19976197205, Loss: 2.2407684326171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11523 , TFLOPS: 100.61096095831087, Tokens per sec: 82211.80615557986, Loss: 2.250624179840088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11524 , TFLOPS: 101.27684280765595, Tokens per sec: 82755.91535600348, Loss: 2.2490525245666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11525 , TFLOPS: 100.58388570590687, Tokens per sec: 82189.68226986156, Loss: 2.276362419128418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11526 , TFLOPS: 100.02465646786611, Tokens per sec: 81732.72166361718, Loss: 2.2753043174743652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11527 , TFLOPS: 100.63742927234868, Tokens per sec: 82233.43409633519, Loss: 2.2573657035827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11528 , TFLOPS: 101.29416569691105, Tokens per sec: 82770.07033474465, Loss: 2.2508206367492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11529 , TFLOPS: 100.07104865380067, Tokens per sec: 81770.62991298542, Loss: 2.2793495655059814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11530 , TFLOPS: 99.24780345970694, Tokens per sec: 81097.93507267482, Loss: 2.245058059692383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11531 , TFLOPS: 100.07362187788762, Tokens per sec: 81772.73256062754, Loss: 2.2473978996276855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11532 , TFLOPS: 99.94996651374927, Tokens per sec: 81671.69057941783, Loss: 2.2437965869903564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11533 , TFLOPS: 101.17893618347259, Tokens per sec: 82675.91333304252, Loss: 2.253831624984741 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11534 , TFLOPS: 101.31461591995203, Tokens per sec: 82786.78073842701, Loss: 2.264680862426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11535 , TFLOPS: 98.09765013844414, Tokens per sec: 80158.11518629054, Loss: 2.2533302307128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11536 , TFLOPS: 100.64029708829901, Tokens per sec: 82235.77746257235, Loss: 2.2614822387695312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11537 , TFLOPS: 99.45200103763851, Tokens per sec: 81264.79017011599, Loss: 2.228869915008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11538 , TFLOPS: 100.01189609843546, Tokens per sec: 81722.29483727425, Loss: 2.268907070159912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11539 , TFLOPS: 99.39494535601737, Tokens per sec: 81218.16850391946, Loss: 2.28070330619812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11540 , TFLOPS: 100.08157448807198, Tokens per sec: 81779.23084312757, Loss: 2.272231340408325 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11541 , TFLOPS: 98.8202069356422, Tokens per sec: 80748.53494554776, Loss: 2.2628040313720703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11542 , TFLOPS: 99.1987006264752, Tokens per sec: 81057.81188362178, Loss: 2.255603313446045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11543 , TFLOPS: 100.72302894164888, Tokens per sec: 82303.3797896519, Loss: 2.2689712047576904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11544 , TFLOPS: 97.86781170113038, Tokens per sec: 79970.30828259373, Loss: 2.273958683013916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11545 , TFLOPS: 100.78293998868432, Tokens per sec: 82352.33464843211, Loss: 2.256730794906616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11546 , TFLOPS: 99.38677731658515, Tokens per sec: 81211.49418863535, Loss: 2.2550268173217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11547 , TFLOPS: 99.95703752531821, Tokens per sec: 81677.46848499487, Loss: 2.2496821880340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11548 , TFLOPS: 99.49756466653712, Tokens per sec: 81302.0213842012, Loss: 2.2353410720825195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11549 , TFLOPS: 100.11339012528985, Tokens per sec: 81805.22821930567, Loss: 2.2746644020080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11550 , TFLOPS: 99.29408915421867, Tokens per sec: 81135.75630516012, Loss: 2.2680790424346924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11551 , TFLOPS: 99.58707379196717, Tokens per sec: 81375.16159475996, Loss: 2.2590982913970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11552 , TFLOPS: 100.10717912886145, Tokens per sec: 81800.15305423875, Loss: 2.2595343589782715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11553 , TFLOPS: 98.70914488561311, Tokens per sec: 80657.78328547748, Loss: 2.2748303413391113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11554 , TFLOPS: 99.46907957573175, Tokens per sec: 81278.74548323266, Loss: 2.255711793899536 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11555 , TFLOPS: 100.72298868665548, Tokens per sec: 82303.34689626054, Loss: 2.2616360187530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11556 , TFLOPS: 100.18599382648664, Tokens per sec: 81864.5545725391, Loss: 2.2614588737487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11557 , TFLOPS: 99.65638330308613, Tokens per sec: 81431.79618048081, Loss: 2.252793073654175 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11558 , TFLOPS: 100.04154579904925, Tokens per sec: 81746.52237089699, Loss: 2.2601675987243652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11559 , TFLOPS: 98.94513460300533, Tokens per sec: 80850.61655847442, Loss: 2.256260871887207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11560 , TFLOPS: 100.20342458849252, Tokens per sec: 81878.79769688184, Loss: 2.2439465522766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11561 , TFLOPS: 100.77487639614212, Tokens per sec: 82345.74567939053, Loss: 2.250424385070801 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11562 , TFLOPS: 101.27468176047503, Tokens per sec: 82754.14950872163, Loss: 2.267427921295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11563 , TFLOPS: 100.60369049925279, Tokens per sec: 82205.86527632525, Loss: 2.260984420776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11564 , TFLOPS: 100.04490531952747, Tokens per sec: 81749.26752156149, Loss: 2.2539474964141846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11565 , TFLOPS: 100.67425394069615, Tokens per sec: 82263.52448079319, Loss: 2.269779920578003 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11566 , TFLOPS: 100.55549431571357, Tokens per sec: 82166.48293407522, Loss: 2.262134313583374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11567 , TFLOPS: 100.65606286736208, Tokens per sec: 82248.66008649205, Loss: 2.2555758953094482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11568 , TFLOPS: 98.8772504901884, Tokens per sec: 80795.14670239919, Loss: 2.2520673274993896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11569 , TFLOPS: 99.46229292701159, Tokens per sec: 81273.19993785955, Loss: 2.2390730381011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11570 , TFLOPS: 101.3841262574262, Tokens per sec: 82843.57942453244, Loss: 2.2790212631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11571 , TFLOPS: 99.84450019631136, Tokens per sec: 81585.51133649479, Loss: 2.2697396278381348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11572 , TFLOPS: 100.55062775308085, Tokens per sec: 82162.50634047155, Loss: 2.25565767288208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11573 , TFLOPS: 99.31587224429973, Tokens per sec: 81153.55582881198, Loss: 2.2638347148895264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11574 , TFLOPS: 99.30280299872479, Tokens per sec: 81142.8766118204, Loss: 2.265277862548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11575 , TFLOPS: 100.53880779829969, Tokens per sec: 82152.84795114717, Loss: 2.27614688873291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11576 , TFLOPS: 98.78721998058576, Tokens per sec: 80721.58045541137, Loss: 2.2548439502716064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11577 , TFLOPS: 100.08305043794229, Tokens per sec: 81780.43687976095, Loss: 2.2597148418426514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11578 , TFLOPS: 99.08782073636962, Tokens per sec: 80967.20907111422, Loss: 2.2425589561462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11579 , TFLOPS: 99.94789872075401, Tokens per sec: 81670.0009325317, Loss: 2.245945692062378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11580 , TFLOPS: 100.03687099124116, Tokens per sec: 81742.70246509674, Loss: 2.2707653045654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11581 , TFLOPS: 100.72280153376737, Tokens per sec: 82303.19396881791, Loss: 2.267026424407959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11582 , TFLOPS: 98.08999900898954, Tokens per sec: 80151.86325145558, Loss: 2.2862603664398193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11583 , TFLOPS: 101.32989639500039, Tokens per sec: 82799.26680794345, Loss: 2.260563850402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11584 , TFLOPS: 98.79595047048929, Tokens per sec: 80728.71436345435, Loss: 2.257185220718384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11585 , TFLOPS: 99.40614871717506, Tokens per sec: 81227.32305872157, Loss: 2.268376111984253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11586 , TFLOPS: 98.61624938134756, Tokens per sec: 80581.87597759906, Loss: 2.275269031524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11587 , TFLOPS: 100.05633336630937, Tokens per sec: 81758.60567277108, Loss: 2.2591795921325684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11588 , TFLOPS: 99.58832488033362, Tokens per sec: 81376.18389127008, Loss: 2.26151180267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11589 , TFLOPS: 98.64289223347336, Tokens per sec: 80603.64653791909, Loss: 2.2635931968688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11590 , TFLOPS: 99.96832198733101, Tokens per sec: 81686.6893093934, Loss: 2.2621119022369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11591 , TFLOPS: 100.23832966522876, Tokens per sec: 81907.31953361964, Loss: 2.279041051864624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11592 , TFLOPS: 99.48761105979702, Tokens per sec: 81293.88803590524, Loss: 2.2599260807037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11593 , TFLOPS: 99.9705818769012, Tokens per sec: 81688.53592333593, Loss: 2.2728400230407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11594 , TFLOPS: 99.97053336994112, Tokens per sec: 81688.49628705018, Loss: 2.253693103790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11595 , TFLOPS: 96.72701095438305, Tokens per sec: 79038.13062560272, Loss: 2.229057550430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11596 , TFLOPS: 100.10367626877478, Tokens per sec: 81797.29077709028, Loss: 2.288740634918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11597 , TFLOPS: 99.40624810374203, Tokens per sec: 81227.40427004387, Loss: 2.2684438228607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11598 , TFLOPS: 99.53485838667376, Tokens per sec: 81332.49504295061, Loss: 2.235814094543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11599 , TFLOPS: 101.26467008501221, Tokens per sec: 82745.96871097751, Loss: 2.2725017070770264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11600 , TFLOPS: 100.4679066427025, Tokens per sec: 82094.91279175058, Loss: 2.2424261569976807 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/model.pt +[rank0]:[2024-08-29 07:25:20,040] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007675808003114071, 'preprocessing_with_comm': 0.001771021998138167, 'state_converting': 2.357598804999725, : 2.3686843489995226}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600, took 14.52s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009600 + eval ppl=7.480604648590088, eval loss=2.0123136043548584 +------------------------------------------------------------------ +iteration: 11601 , TFLOPS: 99.71593094008406, Tokens per sec: 81480.45408756437, Loss: 2.2581982612609863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11602 , TFLOPS: 98.17536200020065, Tokens per sec: 80221.61554901308, Loss: 2.2668912410736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11603 , TFLOPS: 99.95838346034307, Tokens per sec: 81678.56828315112, Loss: 2.262503147125244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11604 , TFLOPS: 98.91132084503658, Tokens per sec: 80822.98646639459, Loss: 2.250678062438965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11605 , TFLOPS: 97.76747408188832, Tokens per sec: 79888.31983099093, Loss: 2.2544403076171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11606 , TFLOPS: 100.50345030337093, Tokens per sec: 82123.95643185798, Loss: 2.247823715209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11607 , TFLOPS: 99.30848791934292, Tokens per sec: 81147.52190680035, Loss: 2.259521961212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11608 , TFLOPS: 97.97408678002603, Tokens per sec: 80057.14838532328, Loss: 2.244039535522461 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11609 , TFLOPS: 101.33116071706401, Tokens per sec: 82800.29991804817, Loss: 2.241701126098633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11610 , TFLOPS: 100.62269112043958, Tokens per sec: 82221.39117301647, Loss: 2.276146173477173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11611 , TFLOPS: 100.58621368338885, Tokens per sec: 82191.58452018956, Loss: 2.254124641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11612 , TFLOPS: 100.62889176371147, Tokens per sec: 82226.4578782527, Loss: 2.245800495147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11613 , TFLOPS: 101.20407621054309, Tokens per sec: 82696.4558963237, Loss: 2.266298294067383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11614 , TFLOPS: 99.46198923771735, Tokens per sec: 81272.95178551954, Loss: 2.2600791454315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11615 , TFLOPS: 101.24114029674962, Tokens per sec: 82726.74191527753, Loss: 2.2612717151641846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11616 , TFLOPS: 99.37468427207308, Tokens per sec: 81201.61265066187, Loss: 2.2576606273651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11617 , TFLOPS: 100.56176394050598, Tokens per sec: 82171.60600588759, Loss: 2.26116943359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11618 , TFLOPS: 100.05326867221481, Tokens per sec: 81756.10143233417, Loss: 2.2373099327087402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11619 , TFLOPS: 99.80748763954352, Tokens per sec: 81555.26742357182, Loss: 2.2834632396698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11620 , TFLOPS: 99.96870573237102, Tokens per sec: 81687.00287734407, Loss: 2.255146026611328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11621 , TFLOPS: 100.12186269595036, Tokens per sec: 81812.15137489574, Loss: 2.235912799835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11622 , TFLOPS: 99.45643409637945, Tokens per sec: 81268.4125365303, Loss: 2.2512528896331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11623 , TFLOPS: 100.8006072538279, Tokens per sec: 82366.7710255768, Loss: 2.2503490447998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11624 , TFLOPS: 98.79910398207373, Tokens per sec: 80731.29117894861, Loss: 2.245182991027832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11625 , TFLOPS: 100.27870136414924, Tokens per sec: 81940.30828806755, Loss: 2.281470775604248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11626 , TFLOPS: 100.73419045266526, Tokens per sec: 82312.50014762624, Loss: 2.2699177265167236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11627 , TFLOPS: 98.63159992127919, Tokens per sec: 80594.41929893526, Loss: 2.250208616256714 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11628 , TFLOPS: 101.26132010161858, Tokens per sec: 82743.23135331033, Loss: 2.27427339553833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11629 , TFLOPS: 100.06708540643031, Tokens per sec: 81767.39144153603, Loss: 2.28012752532959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11630 , TFLOPS: 98.84131774998907, Tokens per sec: 80765.7851353913, Loss: 2.265501022338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11631 , TFLOPS: 100.6630742090978, Tokens per sec: 82254.38923431229, Loss: 2.245028018951416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11632 , TFLOPS: 100.16337551386586, Tokens per sec: 81846.0725670495, Loss: 2.2502942085266113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11633 , TFLOPS: 98.69946526851517, Tokens per sec: 80649.87382117123, Loss: 2.2786834239959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11634 , TFLOPS: 99.32681100913851, Tokens per sec: 81162.49417515061, Loss: 2.2600257396698 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11635 , TFLOPS: 99.4464486441867, Tokens per sec: 81260.25316649521, Loss: 2.234813690185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11636 , TFLOPS: 100.11479194264861, Tokens per sec: 81806.3736803555, Loss: 2.248793363571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11637 , TFLOPS: 98.04236940749475, Tokens per sec: 80112.94387797917, Loss: 2.2610483169555664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11638 , TFLOPS: 100.06332912056165, Tokens per sec: 81764.32208365735, Loss: 2.2342031002044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11639 , TFLOPS: 98.93838483860698, Tokens per sec: 80845.10115223014, Loss: 2.283818006515503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11640 , TFLOPS: 97.7633922232601, Tokens per sec: 79884.98443922943, Loss: 2.251650333404541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11641 , TFLOPS: 101.30680921928504, Tokens per sec: 82780.40168234968, Loss: 2.255347728729248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11642 , TFLOPS: 98.88963391062647, Tokens per sec: 80805.26551401673, Loss: 2.251051902770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11643 , TFLOPS: 99.34393980584501, Tokens per sec: 81176.49055587394, Loss: 2.2552530765533447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11644 , TFLOPS: 100.52808632173986, Tokens per sec: 82144.08715665477, Loss: 2.2524213790893555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11645 , TFLOPS: 100.88537295705274, Tokens per sec: 82436.03526374513, Loss: 2.2490036487579346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11646 , TFLOPS: 99.41620726349062, Tokens per sec: 81235.54215584604, Loss: 2.2527832984924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11647 , TFLOPS: 101.33504440079045, Tokens per sec: 82803.47337599598, Loss: 2.235954523086548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11648 , TFLOPS: 99.9474419762373, Tokens per sec: 81669.62771482943, Loss: 2.2430131435394287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11649 , TFLOPS: 99.86160063628161, Tokens per sec: 81599.48454619898, Loss: 2.2567949295043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11650 , TFLOPS: 99.98802987269747, Tokens per sec: 81702.79312985243, Loss: 2.252967357635498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11651 , TFLOPS: 100.66791870998595, Tokens per sec: 82258.34780069655, Loss: 2.2737927436828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11652 , TFLOPS: 99.28479872593367, Tokens per sec: 81128.16485705155, Loss: 2.2460546493530273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11653 , TFLOPS: 101.28101310865947, Tokens per sec: 82759.32301630653, Loss: 2.26951003074646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11654 , TFLOPS: 100.09167812934248, Tokens per sec: 81787.48678850058, Loss: 2.2738149166107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11655 , TFLOPS: 100.69269324602426, Tokens per sec: 82278.59171184675, Loss: 2.2498836517333984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11656 , TFLOPS: 99.37909951639041, Tokens per sec: 81205.22046045207, Loss: 2.2415883541107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11657 , TFLOPS: 100.51764826519137, Tokens per sec: 82135.55795194951, Loss: 2.248265027999878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11658 , TFLOPS: 100.67689656045611, Tokens per sec: 82265.68383342616, Loss: 2.2752413749694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11659 , TFLOPS: 99.47187206286048, Tokens per sec: 81281.02729635057, Loss: 2.2539925575256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11660 , TFLOPS: 99.23730429567091, Tokens per sec: 81089.35593546845, Loss: 2.2605650424957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11661 , TFLOPS: 100.10051827835129, Tokens per sec: 81794.71030182134, Loss: 2.2525036334991455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11662 , TFLOPS: 99.4689173392331, Tokens per sec: 81278.61291561327, Loss: 2.2706990242004395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11663 , TFLOPS: 100.61264606544056, Tokens per sec: 82213.18309999406, Loss: 2.2361698150634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11664 , TFLOPS: 100.662102400041, Tokens per sec: 82253.59514411539, Loss: 2.253787040710449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11665 , TFLOPS: 98.20948819237628, Tokens per sec: 80249.50093912611, Loss: 2.2616493701934814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11666 , TFLOPS: 101.3024075982842, Tokens per sec: 82776.80500452207, Loss: 2.2745182514190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11667 , TFLOPS: 99.97436531499156, Tokens per sec: 81691.62746799394, Loss: 2.247311592102051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11668 , TFLOPS: 99.38979283810923, Tokens per sec: 81213.9582488991, Loss: 2.2571632862091064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11669 , TFLOPS: 99.47813078954194, Tokens per sec: 81286.14146303594, Loss: 2.255030632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11670 , TFLOPS: 100.15533326751056, Tokens per sec: 81839.50104052674, Loss: 2.24718976020813 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11671 , TFLOPS: 98.73018036146661, Tokens per sec: 80674.97191429869, Loss: 2.264583110809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11672 , TFLOPS: 99.4140511683885, Tokens per sec: 81233.78035503525, Loss: 2.2470853328704834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11673 , TFLOPS: 100.04758802138711, Tokens per sec: 81751.4596263099, Loss: 2.262920618057251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11674 , TFLOPS: 100.13944820013592, Tokens per sec: 81826.52094305691, Loss: 2.2655835151672363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11675 , TFLOPS: 98.7736352345478, Tokens per sec: 80710.48000972129, Loss: 2.2429051399230957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11676 , TFLOPS: 99.86704501732842, Tokens per sec: 81603.93329010316, Loss: 2.2476651668548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11677 , TFLOPS: 98.8375277303682, Tokens per sec: 80762.68821279528, Loss: 2.2652904987335205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11678 , TFLOPS: 99.03120214528983, Tokens per sec: 80920.94456285072, Loss: 2.270528554916382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11679 , TFLOPS: 100.59935553094947, Tokens per sec: 82202.32306213258, Loss: 2.250493049621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11680 , TFLOPS: 98.82841782816993, Tokens per sec: 80755.24427719934, Loss: 2.2400171756744385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11681 , TFLOPS: 98.81595116483383, Tokens per sec: 80745.05744566696, Loss: 2.251221179962158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11682 , TFLOPS: 101.33433589329313, Tokens per sec: 82802.8944362813, Loss: 2.251596689224243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11683 , TFLOPS: 100.14600258025315, Tokens per sec: 81831.87669577546, Loss: 2.257457733154297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11684 , TFLOPS: 99.45946645782801, Tokens per sec: 81270.89035712984, Loss: 2.266155481338501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11685 , TFLOPS: 100.67520832836858, Tokens per sec: 82264.30433552848, Loss: 2.265944480895996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11686 , TFLOPS: 100.63264915596393, Tokens per sec: 82229.52814018607, Loss: 2.2567338943481445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11687 , TFLOPS: 100.06158661130476, Tokens per sec: 81762.89823448737, Loss: 2.243861675262451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11688 , TFLOPS: 100.07843714199612, Tokens per sec: 81776.66723688653, Loss: 2.2870821952819824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11689 , TFLOPS: 101.38668399755576, Tokens per sec: 82845.66941983439, Loss: 2.258427619934082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11690 , TFLOPS: 99.96120825584437, Tokens per sec: 81680.87649627194, Loss: 2.261875867843628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11691 , TFLOPS: 100.62810921232078, Tokens per sec: 82225.81843536669, Loss: 2.2314205169677734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11692 , TFLOPS: 100.02165640808131, Tokens per sec: 81730.27023753861, Loss: 2.2608160972595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11693 , TFLOPS: 99.87675089925479, Tokens per sec: 81611.8642160768, Loss: 2.2685725688934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11694 , TFLOPS: 99.94972817345555, Tokens per sec: 81671.49582542843, Loss: 2.243666410446167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11695 , TFLOPS: 99.65163508677995, Tokens per sec: 81427.91629070707, Loss: 2.2547504901885986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11696 , TFLOPS: 99.73921903217406, Tokens per sec: 81499.48338709993, Loss: 2.2641186714172363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11697 , TFLOPS: 99.51034738170597, Tokens per sec: 81312.46646982204, Loss: 2.2892651557922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11698 , TFLOPS: 99.8187569999655, Tokens per sec: 81564.47590807175, Loss: 2.2589664459228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11699 , TFLOPS: 100.14791227679456, Tokens per sec: 81833.43715797941, Loss: 2.2571191787719727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11700 , TFLOPS: 99.53598278016507, Tokens per sec: 81333.41381381682, Loss: 2.278465509414673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11701 , TFLOPS: 100.60539562937011, Tokens per sec: 82207.25858203806, Loss: 2.241469144821167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11702 , TFLOPS: 99.39777459548245, Tokens per sec: 81220.48034831748, Loss: 2.270944356918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11703 , TFLOPS: 99.36649475535351, Tokens per sec: 81194.9207857334, Loss: 2.2560408115386963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11704 , TFLOPS: 100.58596756043981, Tokens per sec: 82191.38340679214, Loss: 2.2472784519195557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11705 , TFLOPS: 100.72192432555242, Tokens per sec: 82302.47717940414, Loss: 2.2652363777160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11706 , TFLOPS: 98.5947738313859, Tokens per sec: 80564.32775289555, Loss: 2.253443479537964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11707 , TFLOPS: 100.63314868438356, Tokens per sec: 82229.93631771674, Loss: 2.235175132751465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11708 , TFLOPS: 100.64888060835591, Tokens per sec: 82242.79127777032, Loss: 2.256006956100464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11709 , TFLOPS: 98.7538827553929, Tokens per sec: 80694.33975052976, Loss: 2.266763210296631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11710 , TFLOPS: 99.93990006992173, Tokens per sec: 81663.46502903305, Loss: 2.2352147102355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11711 , TFLOPS: 100.03105420147281, Tokens per sec: 81737.94941643953, Loss: 2.2577202320098877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11712 , TFLOPS: 99.47944873940486, Tokens per sec: 81287.21839379598, Loss: 2.2587218284606934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11713 , TFLOPS: 98.33142207848302, Tokens per sec: 80349.13625632139, Loss: 2.2590620517730713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11714 , TFLOPS: 99.86918061613439, Tokens per sec: 81605.67834286271, Loss: 2.2539331912994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11715 , TFLOPS: 99.5139470951238, Tokens per sec: 81315.4078883206, Loss: 2.260524272918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11716 , TFLOPS: 98.40500658633218, Tokens per sec: 80409.2640519187, Loss: 2.256620407104492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11717 , TFLOPS: 101.25968694139745, Tokens per sec: 82741.89685605235, Loss: 2.2656407356262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11718 , TFLOPS: 98.95910313944515, Tokens per sec: 80862.03060918151, Loss: 2.2633254528045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11719 , TFLOPS: 98.7246152147967, Tokens per sec: 80670.42448969508, Loss: 2.272930860519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11720 , TFLOPS: 101.24926855588733, Tokens per sec: 82733.38372505894, Loss: 2.254730463027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11721 , TFLOPS: 100.13460938409493, Tokens per sec: 81822.56702190761, Loss: 2.253251552581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11722 , TFLOPS: 99.52079588622424, Tokens per sec: 81321.00421183233, Loss: 2.2408571243286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11723 , TFLOPS: 100.56360351427988, Tokens per sec: 82173.10916897308, Loss: 2.2571380138397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11724 , TFLOPS: 101.25430589425712, Tokens per sec: 82737.49986391352, Loss: 2.25931978225708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11725 , TFLOPS: 99.18284806605635, Tokens per sec: 81044.85834842233, Loss: 2.242537021636963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11726 , TFLOPS: 99.34895259495931, Tokens per sec: 81180.58663489991, Loss: 2.258113145828247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11727 , TFLOPS: 101.25672023797088, Tokens per sec: 82739.47268631277, Loss: 2.2482497692108154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11728 , TFLOPS: 98.9529816693411, Tokens per sec: 80857.02859837885, Loss: 2.2551259994506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11729 , TFLOPS: 100.63917688868122, Tokens per sec: 82234.86211862815, Loss: 2.2412760257720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11730 , TFLOPS: 100.0683179814454, Tokens per sec: 81768.39861031016, Loss: 2.2560741901397705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11731 , TFLOPS: 100.09354304977461, Tokens per sec: 81789.01066299298, Loss: 2.262153387069702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11732 , TFLOPS: 100.05691158274644, Tokens per sec: 81759.07814830642, Loss: 2.253966808319092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11733 , TFLOPS: 100.00351074779762, Tokens per sec: 81715.44295140996, Loss: 2.255558967590332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11734 , TFLOPS: 100.41159806927298, Tokens per sec: 82048.90160688969, Loss: 2.2504870891571045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11735 , TFLOPS: 99.43349255143603, Tokens per sec: 81249.66640958912, Loss: 2.2493927478790283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11736 , TFLOPS: 100.15055898291877, Tokens per sec: 81835.59984968576, Loss: 2.263495683670044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11737 , TFLOPS: 99.34266603544663, Tokens per sec: 81175.44972529166, Loss: 2.252875328063965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11738 , TFLOPS: 99.46564059438145, Tokens per sec: 81275.93540304409, Loss: 2.2538537979125977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11739 , TFLOPS: 100.6590339494452, Tokens per sec: 82251.08783414462, Loss: 2.266078233718872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11740 , TFLOPS: 100.20491043200042, Tokens per sec: 81880.01181786116, Loss: 2.2779576778411865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11741 , TFLOPS: 99.49141800109011, Tokens per sec: 81296.9987856352, Loss: 2.2461421489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11742 , TFLOPS: 100.69243145922843, Tokens per sec: 82278.37779911688, Loss: 2.243349075317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11743 , TFLOPS: 99.74988163053149, Tokens per sec: 81508.1960706975, Loss: 2.2551233768463135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11744 , TFLOPS: 99.43312539215091, Tokens per sec: 81249.36639428574, Loss: 2.266232490539551 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11745 , TFLOPS: 100.64596557375889, Tokens per sec: 82240.40932796142, Loss: 2.2536392211914062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11746 , TFLOPS: 100.81889877749617, Tokens per sec: 82381.71750043184, Loss: 2.2841992378234863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11747 , TFLOPS: 98.70723566531163, Tokens per sec: 80656.22321242144, Loss: 2.2619545459747314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11748 , TFLOPS: 99.88552581874794, Tokens per sec: 81619.03442868119, Loss: 2.268707752227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11749 , TFLOPS: 100.12110441770577, Tokens per sec: 81811.53176622227, Loss: 2.2798774242401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11750 , TFLOPS: 99.53423163121691, Tokens per sec: 81331.9829049327, Loss: 2.261732339859009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11751 , TFLOPS: 99.38638222194223, Tokens per sec: 81211.17134663198, Loss: 2.2548882961273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11752 , TFLOPS: 99.2491402212903, Tokens per sec: 81099.02737497589, Loss: 2.257483959197998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11753 , TFLOPS: 99.6142220704877, Tokens per sec: 81397.34515199796, Loss: 2.250011682510376 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11754 , TFLOPS: 98.86254658963163, Tokens per sec: 80783.13176674218, Loss: 2.2680797576904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11755 , TFLOPS: 100.58293781862787, Tokens per sec: 82188.9077267651, Loss: 2.2573111057281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11756 , TFLOPS: 98.92413023261582, Tokens per sec: 80833.45334672857, Loss: 2.2581701278686523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11757 , TFLOPS: 99.4687045495556, Tokens per sec: 81278.4390396901, Loss: 2.273195743560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11758 , TFLOPS: 100.5555175848579, Tokens per sec: 82166.50194789204, Loss: 2.264484167098999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11759 , TFLOPS: 100.82761437568469, Tokens per sec: 82388.83924007115, Loss: 2.27278995513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11760 , TFLOPS: 98.98850533199146, Tokens per sec: 80886.05589759088, Loss: 2.269247531890869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11761 , TFLOPS: 101.34766952300514, Tokens per sec: 82813.78968835715, Loss: 2.2585947513580322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11762 , TFLOPS: 100.5366341232932, Tokens per sec: 82151.07178534399, Loss: 2.27457594871521 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11763 , TFLOPS: 99.98084439273381, Tokens per sec: 81696.92168920343, Loss: 2.2596569061279297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11764 , TFLOPS: 99.41593225111862, Tokens per sec: 81235.31743615742, Loss: 2.2794790267944336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11765 , TFLOPS: 101.27904168984124, Tokens per sec: 82757.71211924133, Loss: 2.2508797645568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11766 , TFLOPS: 99.14385028118055, Tokens per sec: 81012.99225450803, Loss: 2.282644271850586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11767 , TFLOPS: 100.16655193152378, Tokens per sec: 81848.66809968605, Loss: 2.289274215698242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11768 , TFLOPS: 100.08362158928253, Tokens per sec: 81780.9035822238, Loss: 2.2597856521606445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11769 , TFLOPS: 99.40338947653281, Tokens per sec: 81225.06841216361, Loss: 2.267801284790039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11770 , TFLOPS: 99.3904171852282, Tokens per sec: 81214.46841900214, Loss: 2.272167682647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11771 , TFLOPS: 100.01625572600045, Tokens per sec: 81725.85720118442, Loss: 2.278559923171997 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11772 , TFLOPS: 100.63623266322593, Tokens per sec: 82232.45631621752, Loss: 2.2419822216033936 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11773 , TFLOPS: 99.56403541427441, Tokens per sec: 81356.33634328662, Loss: 2.236598491668701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11774 , TFLOPS: 99.99590742452712, Tokens per sec: 81709.23008023867, Loss: 2.241922616958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11775 , TFLOPS: 100.13594338122297, Tokens per sec: 81823.65706530107, Loss: 2.260331153869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11776 , TFLOPS: 99.50278039622232, Tokens per sec: 81306.28329119181, Loss: 2.2569656372070312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11777 , TFLOPS: 100.7881134282875, Tokens per sec: 82356.56199910805, Loss: 2.2589237689971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11778 , TFLOPS: 99.58038447865579, Tokens per sec: 81369.695584655, Loss: 2.2450952529907227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11779 , TFLOPS: 98.92235292301918, Tokens per sec: 80832.00106130513, Loss: 2.2596371173858643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11780 , TFLOPS: 100.27988700584926, Tokens per sec: 81941.27710642175, Loss: 2.261096477508545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11781 , TFLOPS: 100.65214480032836, Tokens per sec: 82245.45853305876, Loss: 2.271238088607788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11782 , TFLOPS: 99.42731868641353, Tokens per sec: 81244.62158554982, Loss: 2.2368669509887695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11783 , TFLOPS: 100.05715844052631, Tokens per sec: 81759.27986215304, Loss: 2.282881498336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11784 , TFLOPS: 101.31899295775665, Tokens per sec: 82790.35732869215, Loss: 2.2359158992767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11785 , TFLOPS: 97.40769239878816, Tokens per sec: 79594.33295612533, Loss: 2.2512147426605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11786 , TFLOPS: 100.66940797159708, Tokens per sec: 82259.564714696, Loss: 2.273815631866455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11787 , TFLOPS: 100.04746084753744, Tokens per sec: 81751.35570928357, Loss: 2.258451461791992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11788 , TFLOPS: 99.99728363967945, Tokens per sec: 81710.35462106658, Loss: 2.2457284927368164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11789 , TFLOPS: 99.24814850987505, Tokens per sec: 81098.21702204956, Loss: 2.23004412651062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11790 , TFLOPS: 100.05965978418841, Tokens per sec: 81761.32377444966, Loss: 2.2598907947540283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11791 , TFLOPS: 100.08221489923127, Tokens per sec: 81779.7541395715, Loss: 2.2574703693389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11792 , TFLOPS: 98.9057955145863, Tokens per sec: 80818.471576649, Loss: 2.247213840484619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11793 , TFLOPS: 100.5152319493653, Tokens per sec: 82133.58351808653, Loss: 2.2553744316101074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11794 , TFLOPS: 99.60673471731204, Tokens per sec: 81391.2270429765, Loss: 2.2601523399353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11795 , TFLOPS: 98.77989070810571, Tokens per sec: 80715.59151819567, Loss: 2.233365297317505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11796 , TFLOPS: 99.90159502415857, Tokens per sec: 81632.16499008036, Loss: 2.243208169937134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11797 , TFLOPS: 100.26987517098553, Tokens per sec: 81933.09617842705, Loss: 2.262338161468506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11798 , TFLOPS: 100.06618084843491, Tokens per sec: 81766.6523039126, Loss: 2.25477933883667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11799 , TFLOPS: 99.88660680894178, Tokens per sec: 81619.91773359574, Loss: 2.246770143508911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11800 , TFLOPS: 101.23098776748579, Tokens per sec: 82718.44602226668, Loss: 2.2662370204925537 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/optimizer.pt +[rank0]:[2024-08-29 08:19:15,754] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007618864001415204, 'preprocessing_with_comm': 0.0016802960017230362, 'state_converting': 2.3782381279961555, : 2.3891410420037573}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011800, took 14.35s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0009800 + eval ppl=8.077244758605957, eval loss=2.089050769805908 +------------------------------------------------------------------ +iteration: 11801 , TFLOPS: 98.00176238846822, Tokens per sec: 80079.76283741494, Loss: 2.2493412494659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11802 , TFLOPS: 99.69595670624935, Tokens per sec: 81464.1326269155, Loss: 2.2727203369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11803 , TFLOPS: 99.5094465334823, Tokens per sec: 81311.73036354857, Loss: 2.250481367111206 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11804 , TFLOPS: 99.63589510642362, Tokens per sec: 81415.05474757477, Loss: 2.287107229232788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11805 , TFLOPS: 99.64147565029864, Tokens per sec: 81419.61475363103, Loss: 2.258300304412842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11806 , TFLOPS: 100.14831247488277, Tokens per sec: 81833.7641701393, Loss: 2.2554738521575928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11807 , TFLOPS: 100.62215765692389, Tokens per sec: 82220.95526624523, Loss: 2.265429973602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11808 , TFLOPS: 100.13435385084844, Tokens per sec: 81822.3582191139, Loss: 2.2470715045928955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11809 , TFLOPS: 99.9591791466412, Tokens per sec: 81679.2184589081, Loss: 2.2752726078033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11810 , TFLOPS: 100.25028265284469, Tokens per sec: 81917.08662749788, Loss: 2.2603304386138916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11811 , TFLOPS: 101.34005779843852, Tokens per sec: 82807.56995227054, Loss: 2.2555580139160156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11812 , TFLOPS: 101.23520128386536, Tokens per sec: 82721.888994966, Loss: 2.2385051250457764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11813 , TFLOPS: 99.69163835116667, Tokens per sec: 81460.6039878131, Loss: 2.273365020751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11814 , TFLOPS: 100.30097152152459, Tokens per sec: 81958.50580694378, Loss: 2.271444320678711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11815 , TFLOPS: 100.41828994085444, Tokens per sec: 82054.369707423, Loss: 2.2837963104248047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11816 , TFLOPS: 100.55073068179408, Tokens per sec: 82162.59044617275, Loss: 2.245532751083374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11817 , TFLOPS: 98.69956887123422, Tokens per sec: 80649.95847761995, Loss: 2.2523722648620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11818 , TFLOPS: 101.25940201211833, Tokens per sec: 82741.66403300366, Loss: 2.2261767387390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11819 , TFLOPS: 100.10361260422738, Tokens per sec: 81797.23875514974, Loss: 2.2554171085357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11820 , TFLOPS: 99.76201696603923, Tokens per sec: 81518.1121657322, Loss: 2.2799479961395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11821 , TFLOPS: 99.89512965635922, Tokens per sec: 81626.88197161813, Loss: 2.2419443130493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11822 , TFLOPS: 99.96399638591292, Tokens per sec: 81683.1547491238, Loss: 2.251554489135742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11823 , TFLOPS: 100.08123868999559, Tokens per sec: 81778.95645387514, Loss: 2.2549479007720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11824 , TFLOPS: 99.94894438430171, Tokens per sec: 81670.8553711343, Loss: 2.2535324096679688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11825 , TFLOPS: 99.6007181745159, Tokens per sec: 81386.31077097819, Loss: 2.258423089981079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11826 , TFLOPS: 99.36984505421476, Tokens per sec: 81197.6584011773, Loss: 2.2705349922180176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11827 , TFLOPS: 99.53705833436553, Tokens per sec: 81334.29267684124, Loss: 2.268261432647705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11828 , TFLOPS: 99.16134032576623, Tokens per sec: 81027.2838201728, Loss: 2.2564384937286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11829 , TFLOPS: 100.78477524491603, Tokens per sec: 82353.83428354273, Loss: 2.247972011566162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11830 , TFLOPS: 98.34660238419704, Tokens per sec: 80361.54047489616, Loss: 2.2517189979553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11831 , TFLOPS: 100.01106626370995, Tokens per sec: 81721.6167579582, Loss: 2.2617123126983643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11832 , TFLOPS: 99.60508307463165, Tokens per sec: 81389.8774432253, Loss: 2.2668585777282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11833 , TFLOPS: 98.38456090881229, Tokens per sec: 80392.55736249863, Loss: 2.249555826187134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11834 , TFLOPS: 99.38071185907006, Tokens per sec: 81206.537947161, Loss: 2.2401559352874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11835 , TFLOPS: 100.09841427271739, Tokens per sec: 81792.99106465597, Loss: 2.26702880859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11836 , TFLOPS: 96.98918290262843, Tokens per sec: 79252.35807342037, Loss: 2.2545108795166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11837 , TFLOPS: 98.57440598296823, Tokens per sec: 80547.68465963843, Loss: 2.262054443359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11838 , TFLOPS: 99.93710308884184, Tokens per sec: 81661.17954379196, Loss: 2.262512683868408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11839 , TFLOPS: 99.43816543916464, Tokens per sec: 81253.484746443, Loss: 2.2496039867401123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11840 , TFLOPS: 99.94376166816707, Tokens per sec: 81666.62044036569, Loss: 2.247929573059082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11841 , TFLOPS: 98.9050125316252, Tokens per sec: 80817.83178111573, Loss: 2.266286611557007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11842 , TFLOPS: 100.22538836065644, Tokens per sec: 81896.74486051477, Loss: 2.2630796432495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11843 , TFLOPS: 99.37457123314087, Tokens per sec: 81201.52028364048, Loss: 2.2672739028930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11844 , TFLOPS: 100.19758665092775, Tokens per sec: 81874.02736781578, Loss: 2.257017135620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11845 , TFLOPS: 100.26052781343598, Tokens per sec: 81925.45821195093, Loss: 2.2383599281311035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11846 , TFLOPS: 100.1202992778633, Tokens per sec: 81810.87386573084, Loss: 2.252751588821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11847 , TFLOPS: 99.99824763055608, Tokens per sec: 81711.14232282719, Loss: 2.2589449882507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11848 , TFLOPS: 101.32058507244005, Tokens per sec: 82791.65828658463, Loss: 2.2441885471343994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11849 , TFLOPS: 100.70802178678461, Tokens per sec: 82291.11705709377, Loss: 2.2891180515289307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11850 , TFLOPS: 101.33363389967744, Tokens per sec: 82802.320819227, Loss: 2.2536067962646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11851 , TFLOPS: 100.71473067476265, Tokens per sec: 82296.59906216277, Loss: 2.2504894733428955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11852 , TFLOPS: 100.48478446286931, Tokens per sec: 82108.70409308263, Loss: 2.2663378715515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11853 , TFLOPS: 99.59201739415015, Tokens per sec: 81379.20113936333, Loss: 2.2573678493499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11854 , TFLOPS: 100.55498089121478, Tokens per sec: 82166.06340170054, Loss: 2.234483003616333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11855 , TFLOPS: 100.09426003337421, Tokens per sec: 81789.59652874903, Loss: 2.263564109802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11856 , TFLOPS: 99.43909379398835, Tokens per sec: 81254.24332906744, Loss: 2.268885612487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11857 , TFLOPS: 100.62188924362562, Tokens per sec: 82220.73593882963, Loss: 2.2545785903930664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11858 , TFLOPS: 100.0440466364657, Tokens per sec: 81748.56586952723, Loss: 2.2697296142578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11859 , TFLOPS: 100.70412956156616, Tokens per sec: 82287.93661967298, Loss: 2.2661983966827393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11860 , TFLOPS: 99.12557758229966, Tokens per sec: 80998.06116187133, Loss: 2.279924154281616 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11861 , TFLOPS: 99.44265157335055, Tokens per sec: 81257.15048216967, Loss: 2.2520549297332764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11862 , TFLOPS: 100.60292880072525, Tokens per sec: 82205.24287285053, Loss: 2.2421982288360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11863 , TFLOPS: 99.16601048868968, Tokens per sec: 81031.09993051823, Loss: 2.258742094039917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11864 , TFLOPS: 99.3060446889371, Tokens per sec: 81145.5254803414, Loss: 2.2451117038726807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11865 , TFLOPS: 99.45115810768925, Tokens per sec: 81264.10139035551, Loss: 2.2675693035125732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11866 , TFLOPS: 98.69557345512649, Tokens per sec: 80646.69372026715, Loss: 2.255279779434204 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11867 , TFLOPS: 100.63945761904205, Tokens per sec: 82235.09151063261, Loss: 2.2661337852478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11868 , TFLOPS: 98.39800547753653, Tokens per sec: 80403.5432656972, Loss: 2.252260208129883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11869 , TFLOPS: 99.59300415341247, Tokens per sec: 81380.007445758, Loss: 2.2447028160095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11870 , TFLOPS: 100.1387052752961, Tokens per sec: 81825.91388004571, Loss: 2.2621724605560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11871 , TFLOPS: 98.1135240804942, Tokens per sec: 80171.08619297126, Loss: 2.2450294494628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11872 , TFLOPS: 100.68006481156668, Tokens per sec: 82268.27269296662, Loss: 2.2732694149017334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11873 , TFLOPS: 99.28487691028732, Tokens per sec: 81128.22874349957, Loss: 2.2762818336486816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11874 , TFLOPS: 99.59833132095248, Tokens per sec: 81384.36041148845, Loss: 2.2554233074188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11875 , TFLOPS: 99.49284177519569, Tokens per sec: 81298.16218810808, Loss: 2.2490451335906982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11876 , TFLOPS: 99.57385737030435, Tokens per sec: 81364.36211639832, Loss: 2.255557060241699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11877 , TFLOPS: 99.29783523601071, Tokens per sec: 81138.81732502513, Loss: 2.243415355682373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11878 , TFLOPS: 100.10699609868504, Tokens per sec: 81800.00349557, Loss: 2.284358501434326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11879 , TFLOPS: 99.99198195958347, Tokens per sec: 81706.02248178277, Loss: 2.263658046722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11880 , TFLOPS: 100.62254535975478, Tokens per sec: 82221.27206820872, Loss: 2.250200033187866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11881 , TFLOPS: 100.27643483126243, Tokens per sec: 81938.45624569998, Loss: 2.2702691555023193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11882 , TFLOPS: 100.65107698294709, Tokens per sec: 82244.58599198847, Loss: 2.266139268875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11883 , TFLOPS: 99.99016227400101, Tokens per sec: 81704.53556985046, Loss: 2.2922260761260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11884 , TFLOPS: 99.53059814531413, Tokens per sec: 81329.01389006729, Loss: 2.255915641784668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11885 , TFLOPS: 99.93825028603678, Tokens per sec: 81662.11694815144, Loss: 2.2529101371765137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11886 , TFLOPS: 101.31717423292959, Tokens per sec: 82788.8712018178, Loss: 2.248333692550659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11887 , TFLOPS: 99.9882762288339, Tokens per sec: 81702.99443379328, Loss: 2.2520620822906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11888 , TFLOPS: 101.35710471095656, Tokens per sec: 82821.49942331559, Loss: 2.2676875591278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11889 , TFLOPS: 100.70091859199239, Tokens per sec: 82285.31285377654, Loss: 2.2297492027282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11890 , TFLOPS: 100.62772758390987, Tokens per sec: 82225.50659696816, Loss: 2.253999710083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11891 , TFLOPS: 100.03778901819544, Tokens per sec: 81743.4526085532, Loss: 2.258265733718872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11892 , TFLOPS: 100.6120911909373, Tokens per sec: 82212.7296977537, Loss: 2.270048141479492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11893 , TFLOPS: 99.3819616923824, Tokens per sec: 81207.55921813408, Loss: 2.2861034870147705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11894 , TFLOPS: 100.1589223035095, Tokens per sec: 81842.43373423148, Loss: 2.258394479751587 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11895 , TFLOPS: 100.71605938234823, Tokens per sec: 82297.68478333493, Loss: 2.284346342086792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11896 , TFLOPS: 100.02840339478811, Tokens per sec: 81735.78337405963, Loss: 2.248953104019165 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11897 , TFLOPS: 100.71827598199144, Tokens per sec: 82299.49602396373, Loss: 2.2639384269714355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11898 , TFLOPS: 99.32408413321187, Tokens per sec: 81160.26597463492, Loss: 2.251364231109619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11899 , TFLOPS: 99.49819800771478, Tokens per sec: 81302.53890358111, Loss: 2.2583107948303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11900 , TFLOPS: 100.62647798595225, Tokens per sec: 82224.48551830948, Loss: 2.2662675380706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11901 , TFLOPS: 100.28508567562247, Tokens per sec: 81945.52507331912, Loss: 2.2719767093658447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11902 , TFLOPS: 98.59808953562934, Tokens per sec: 80567.0371001867, Loss: 2.2686402797698975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11903 , TFLOPS: 99.55027732677628, Tokens per sec: 81345.0942558272, Loss: 2.2553353309631348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11904 , TFLOPS: 99.49287415419008, Tokens per sec: 81298.1886458179, Loss: 2.2591307163238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11905 , TFLOPS: 100.00063727945111, Tokens per sec: 81713.09496645437, Loss: 2.2521235942840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11906 , TFLOPS: 98.41125606586078, Tokens per sec: 80414.37066251744, Loss: 2.243340492248535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11907 , TFLOPS: 99.94698738849112, Tokens per sec: 81669.25625947991, Loss: 2.267327308654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11908 , TFLOPS: 99.40521623863722, Tokens per sec: 81226.56110650417, Loss: 2.2491180896759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11909 , TFLOPS: 98.91576914038868, Tokens per sec: 80826.62128303609, Loss: 2.2551164627075195 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11910 , TFLOPS: 100.02502683719607, Tokens per sec: 81733.02430193084, Loss: 2.2296953201293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11911 , TFLOPS: 100.00505756484185, Tokens per sec: 81716.7068954354, Loss: 2.2573437690734863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11912 , TFLOPS: 99.05432797441912, Tokens per sec: 80939.84127314408, Loss: 2.2486190795898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11913 , TFLOPS: 100.02618485251331, Tokens per sec: 81733.97054605653, Loss: 2.2544217109680176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11914 , TFLOPS: 99.35545565464297, Tokens per sec: 81185.90045237108, Loss: 2.220240592956543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11915 , TFLOPS: 99.36876597696832, Tokens per sec: 81196.7766593813, Loss: 2.257843017578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11916 , TFLOPS: 100.13963509637153, Tokens per sec: 81826.67366078217, Loss: 2.26739501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11917 , TFLOPS: 99.59173684936059, Tokens per sec: 81378.9718989939, Loss: 2.2521212100982666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11918 , TFLOPS: 100.03162870434176, Tokens per sec: 81738.41885752277, Loss: 2.2290284633636475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11919 , TFLOPS: 99.31256904384408, Tokens per sec: 81150.85669868773, Loss: 2.268357753753662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11920 , TFLOPS: 100.7148268704989, Tokens per sec: 82296.67766617515, Loss: 2.2558164596557617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11921 , TFLOPS: 100.13995381760405, Tokens per sec: 81826.93409610588, Loss: 2.275416851043701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11922 , TFLOPS: 99.40497828608352, Tokens per sec: 81226.3666693471, Loss: 2.281432628631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11923 , TFLOPS: 99.84551973741871, Tokens per sec: 81586.34442977892, Loss: 2.2715554237365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11924 , TFLOPS: 101.32038256603185, Tokens per sec: 82791.49281338551, Loss: 2.2472403049468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11925 , TFLOPS: 100.65969072680525, Tokens per sec: 82251.62450383247, Loss: 2.270425319671631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11926 , TFLOPS: 100.7462543938726, Tokens per sec: 82322.35790454023, Loss: 2.260533094406128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11927 , TFLOPS: 100.64037439420747, Tokens per sec: 82235.84063122018, Loss: 2.2654950618743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11928 , TFLOPS: 100.63646101129761, Tokens per sec: 82232.6429053051, Loss: 2.273538589477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11929 , TFLOPS: 100.01526998780466, Tokens per sec: 81725.05172912948, Loss: 2.2639498710632324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11930 , TFLOPS: 99.967861464756, Tokens per sec: 81686.31300454245, Loss: 2.252096652984619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11931 , TFLOPS: 100.08926758107764, Tokens per sec: 81785.51706746034, Loss: 2.269056558609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11932 , TFLOPS: 99.90494241026565, Tokens per sec: 81634.90022543784, Loss: 2.265374183654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11933 , TFLOPS: 100.04165048068216, Tokens per sec: 81746.60790895399, Loss: 2.2528421878814697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11934 , TFLOPS: 100.00934175160224, Tokens per sec: 81720.2076147221, Loss: 2.254153251647949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11935 , TFLOPS: 100.64687997544166, Tokens per sec: 82241.1565091152, Loss: 2.246760606765747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11936 , TFLOPS: 99.80944816016748, Tokens per sec: 81556.86941544194, Loss: 2.2528984546661377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11937 , TFLOPS: 98.3414696916761, Tokens per sec: 80357.34641971004, Loss: 2.263355016708374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11938 , TFLOPS: 101.2772012546189, Tokens per sec: 82756.20825224415, Loss: 2.2567543983459473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11939 , TFLOPS: 99.6108255424765, Tokens per sec: 81394.56976152559, Loss: 2.26137113571167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11940 , TFLOPS: 98.55183909274895, Tokens per sec: 80529.24467271683, Loss: 2.262650966644287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11941 , TFLOPS: 99.59959535153969, Tokens per sec: 81385.3932834203, Loss: 2.251605987548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11942 , TFLOPS: 99.51294214088405, Tokens per sec: 81314.58671434148, Loss: 2.2452354431152344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11943 , TFLOPS: 100.03450707025549, Tokens per sec: 81740.77084440661, Loss: 2.22057843208313 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11944 , TFLOPS: 97.91684234183658, Tokens per sec: 80010.37248127574, Loss: 2.2608885765075684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11945 , TFLOPS: 100.14256441422016, Tokens per sec: 81829.06728180363, Loss: 2.2608399391174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11946 , TFLOPS: 99.5768832917791, Tokens per sec: 81366.83467472949, Loss: 2.2728779315948486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11947 , TFLOPS: 98.70064901376281, Tokens per sec: 80650.8410898854, Loss: 2.273893356323242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11948 , TFLOPS: 100.74648790037166, Tokens per sec: 82322.5487087116, Loss: 2.247710943222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11949 , TFLOPS: 99.9001411849282, Tokens per sec: 81630.97702062008, Loss: 2.253815174102783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11950 , TFLOPS: 100.05790231333633, Tokens per sec: 81759.88769977509, Loss: 2.2512400150299072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11951 , TFLOPS: 99.43243936955324, Tokens per sec: 81248.80582756139, Loss: 2.279933452606201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11952 , TFLOPS: 100.78366182195501, Tokens per sec: 82352.92447697923, Loss: 2.255525588989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11953 , TFLOPS: 98.98418167128614, Tokens per sec: 80882.52292312763, Loss: 2.24993896484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11954 , TFLOPS: 99.30195840138032, Tokens per sec: 81142.18646958836, Loss: 2.246063709259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11955 , TFLOPS: 99.93746268964291, Tokens per sec: 81661.47338286348, Loss: 2.251983642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11956 , TFLOPS: 100.65703669264028, Tokens per sec: 82249.45582419535, Loss: 2.2519960403442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11957 , TFLOPS: 100.21210684657785, Tokens per sec: 81885.89219346456, Loss: 2.256007432937622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11958 , TFLOPS: 99.53756197194723, Tokens per sec: 81334.70421207404, Loss: 2.2589399814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11959 , TFLOPS: 100.76360255938407, Tokens per sec: 82336.53353716116, Loss: 2.249967575073242 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11960 , TFLOPS: 100.12946080261126, Tokens per sec: 81818.35998344103, Loss: 2.285433292388916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11961 , TFLOPS: 99.33726258741247, Tokens per sec: 81171.03442880583, Loss: 2.254990577697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11962 , TFLOPS: 100.77116855980861, Tokens per sec: 82342.71591087423, Loss: 2.2504749298095703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11963 , TFLOPS: 101.31413754001035, Tokens per sec: 82786.38984186208, Loss: 2.2609543800354004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11964 , TFLOPS: 99.87182586041624, Tokens per sec: 81607.83984006006, Loss: 2.249814987182617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11965 , TFLOPS: 100.07898958922863, Tokens per sec: 81777.11865574146, Loss: 2.241959810256958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11966 , TFLOPS: 101.26707884902294, Tokens per sec: 82747.93697405778, Loss: 2.2427265644073486 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11967 , TFLOPS: 99.2492092268935, Tokens per sec: 81099.08376123063, Loss: 2.24745512008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11968 , TFLOPS: 99.64967904778945, Tokens per sec: 81426.317960895, Loss: 2.252572774887085 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11969 , TFLOPS: 99.0016817761355, Tokens per sec: 80896.82270929281, Loss: 2.2660374641418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11970 , TFLOPS: 100.58958010951756, Tokens per sec: 82194.33531363883, Loss: 2.278534173965454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11971 , TFLOPS: 99.52437096410023, Tokens per sec: 81323.9254999955, Loss: 2.239581346511841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11972 , TFLOPS: 100.03481596723869, Tokens per sec: 81741.0232520833, Loss: 2.2548940181732178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11973 , TFLOPS: 100.0274443497533, Tokens per sec: 81734.99971367365, Loss: 2.2473416328430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11974 , TFLOPS: 98.88095913833362, Tokens per sec: 80798.17713426774, Loss: 2.2566096782684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11975 , TFLOPS: 98.97605364298154, Tokens per sec: 80875.88130196588, Loss: 2.2684271335601807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11976 , TFLOPS: 100.23545117186619, Tokens per sec: 81904.96744259408, Loss: 2.262211799621582 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11977 , TFLOPS: 100.19747587140897, Tokens per sec: 81873.93684701926, Loss: 2.251331090927124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11978 , TFLOPS: 98.33209001150651, Tokens per sec: 80349.68204158903, Loss: 2.276549816131592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11979 , TFLOPS: 100.0262390778008, Tokens per sec: 81734.01485493481, Loss: 2.2688844203948975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11980 , TFLOPS: 100.04512765693933, Tokens per sec: 81749.4491991842, Loss: 2.2750606536865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11981 , TFLOPS: 99.91272862480145, Tokens per sec: 81641.26254177003, Loss: 2.247915029525757 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11982 , TFLOPS: 98.9834501018731, Tokens per sec: 80881.92513892797, Loss: 2.2664363384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11983 , TFLOPS: 100.67384902321832, Tokens per sec: 82263.19361229856, Loss: 2.2618441581726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11984 , TFLOPS: 99.34609758174071, Tokens per sec: 81178.25373010442, Loss: 2.2677576541900635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11985 , TFLOPS: 98.83500543423614, Tokens per sec: 80760.62717969611, Loss: 2.2507753372192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11986 , TFLOPS: 99.2922534752058, Tokens per sec: 81134.25632458413, Loss: 2.27783203125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11987 , TFLOPS: 99.39104569918396, Tokens per sec: 81214.98199393478, Loss: 2.248965263366699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11988 , TFLOPS: 99.6002771088129, Tokens per sec: 81385.95036483822, Loss: 2.235543727874756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11989 , TFLOPS: 99.47932190873502, Tokens per sec: 81287.11475719073, Loss: 2.258483409881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11990 , TFLOPS: 99.55657315926156, Tokens per sec: 81350.23874262009, Loss: 2.2564315795898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11991 , TFLOPS: 98.40488419334237, Tokens per sec: 80409.16404145604, Loss: 2.2491869926452637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11992 , TFLOPS: 99.97672404745049, Tokens per sec: 81693.5548490053, Loss: 2.2460005283355713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11993 , TFLOPS: 98.84438796466725, Tokens per sec: 80768.29388683918, Loss: 2.2758166790008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11994 , TFLOPS: 100.03225618676656, Tokens per sec: 81738.9315895649, Loss: 2.2588376998901367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11995 , TFLOPS: 100.15809074566265, Tokens per sec: 81841.75424690862, Loss: 2.2537906169891357 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11996 , TFLOPS: 98.95112496803208, Tokens per sec: 80855.51143994348, Loss: 2.268695831298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11997 , TFLOPS: 100.22187856821503, Tokens per sec: 81893.8769187608, Loss: 2.260355234146118 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11998 , TFLOPS: 100.15154410379634, Tokens per sec: 81836.4048173141, Loss: 2.2673401832580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11999 , TFLOPS: 99.47950104840899, Tokens per sec: 81287.26113682981, Loss: 2.2790496349334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12000 , TFLOPS: 99.98463557478955, Tokens per sec: 81700.01956165487, Loss: 2.2484688758850098 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/model.pt +[rank0]:[2024-08-29 09:13:12,435] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007772296994517092, 'preprocessing_with_comm': 0.0017129279949585907, 'state_converting': 2.43907166000281, : 2.4502057540012174}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012000, took 14.87s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010000 + eval ppl=7.915558815002441, eval loss=2.0688302516937256 +------------------------------------------------------------------ +iteration: 12001 , TFLOPS: 100.47018537276132, Tokens per sec: 82096.77480074138, Loss: 2.242800712585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12002 , TFLOPS: 96.20769944320149, Tokens per sec: 78613.7878215487, Loss: 2.2672111988067627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12003 , TFLOPS: 100.6618018788638, Tokens per sec: 82253.34958052539, Loss: 2.2668652534484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12004 , TFLOPS: 98.59831626014602, Tokens per sec: 80567.22236262567, Loss: 2.270634651184082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12005 , TFLOPS: 98.67648702747604, Tokens per sec: 80631.09770890563, Loss: 2.2513256072998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12006 , TFLOPS: 99.93557142917729, Tokens per sec: 81659.92798525163, Loss: 2.2527568340301514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12007 , TFLOPS: 99.64069871019937, Tokens per sec: 81418.9798958757, Loss: 2.237581968307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12008 , TFLOPS: 99.53644019609713, Tokens per sec: 81333.78758014984, Loss: 2.2580959796905518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12009 , TFLOPS: 101.34533640380658, Tokens per sec: 82811.88323660017, Loss: 2.2718067169189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12010 , TFLOPS: 98.75568588095803, Tokens per sec: 80695.8131308456, Loss: 2.2510628700256348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12011 , TFLOPS: 101.09562898171059, Tokens per sec: 82607.84087396464, Loss: 2.2319562435150146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12012 , TFLOPS: 100.60753980058348, Tokens per sec: 82209.01063953238, Loss: 2.224160671234131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12013 , TFLOPS: 99.36663649383698, Tokens per sec: 81195.03660389692, Loss: 2.2625958919525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12014 , TFLOPS: 100.96207165074043, Tokens per sec: 82498.70774075735, Loss: 2.243985414505005 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12015 , TFLOPS: 100.04597292467106, Tokens per sec: 81750.13988920693, Loss: 2.2615246772766113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12016 , TFLOPS: 100.06506499902066, Tokens per sec: 81765.74051463169, Loss: 2.2461557388305664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12017 , TFLOPS: 99.56771738415972, Tokens per sec: 81359.34497566224, Loss: 2.264240026473999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12018 , TFLOPS: 101.34752456514165, Tokens per sec: 82813.67123955535, Loss: 2.263620615005493 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12019 , TFLOPS: 100.06928801369928, Tokens per sec: 81769.19124863566, Loss: 2.2633328437805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12020 , TFLOPS: 100.08178822369662, Tokens per sec: 81779.40549200852, Loss: 2.253171920776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12021 , TFLOPS: 99.50511649667268, Tokens per sec: 81308.19217900638, Loss: 2.2522149085998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12022 , TFLOPS: 100.69971043714784, Tokens per sec: 82284.32563935239, Loss: 2.2626569271087646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12023 , TFLOPS: 98.8752044980546, Tokens per sec: 80793.47486955814, Loss: 2.2415404319763184 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12024 , TFLOPS: 99.52392473851954, Tokens per sec: 81323.56087758666, Loss: 2.2366907596588135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12025 , TFLOPS: 100.70220593260846, Tokens per sec: 82286.36477293304, Loss: 2.255558490753174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12026 , TFLOPS: 99.44774751031869, Tokens per sec: 81261.31450344744, Loss: 2.2618720531463623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12027 , TFLOPS: 99.41930752900846, Tokens per sec: 81238.0754626086, Loss: 2.262981414794922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12028 , TFLOPS: 101.26012228894783, Tokens per sec: 82742.25258974261, Loss: 2.250074625015259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12029 , TFLOPS: 98.15391806802434, Tokens per sec: 80204.09316001588, Loss: 2.260481834411621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12030 , TFLOPS: 100.12344621687288, Tokens per sec: 81813.44531060499, Loss: 2.275557041168213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12031 , TFLOPS: 100.87457367703153, Tokens per sec: 82427.21089404161, Loss: 2.2449254989624023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12032 , TFLOPS: 98.93207977193197, Tokens per sec: 80839.94911994314, Loss: 2.265018939971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12033 , TFLOPS: 99.40102734466583, Tokens per sec: 81223.13825340843, Loss: 2.2488465309143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12034 , TFLOPS: 98.30126930434314, Tokens per sec: 80324.49764837025, Loss: 2.24834942817688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12035 , TFLOPS: 100.05092563468423, Tokens per sec: 81754.18687605279, Loss: 2.2278013229370117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12036 , TFLOPS: 98.41125472077164, Tokens per sec: 80414.36956341047, Loss: 2.259657859802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12037 , TFLOPS: 99.48811654577669, Tokens per sec: 81294.3010815116, Loss: 2.279585123062134 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12038 , TFLOPS: 98.33109513997103, Tokens per sec: 80348.8691064471, Loss: 2.2624258995056152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12039 , TFLOPS: 100.23273471516518, Tokens per sec: 81902.74775589585, Loss: 2.2582733631134033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12040 , TFLOPS: 97.21707840881291, Tokens per sec: 79438.57735807594, Loss: 2.2831437587738037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12041 , TFLOPS: 100.25762265363383, Tokens per sec: 81923.08433109112, Loss: 2.252708673477173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12042 , TFLOPS: 100.03996344022869, Tokens per sec: 81745.22938477117, Loss: 2.2600393295288086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12043 , TFLOPS: 98.9444164393916, Tokens per sec: 80850.02972849848, Loss: 2.2773220539093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12044 , TFLOPS: 100.7330083013331, Tokens per sec: 82311.53418134144, Loss: 2.27980637550354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12045 , TFLOPS: 100.18190914295835, Tokens per sec: 81861.21687247902, Loss: 2.239496946334839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12046 , TFLOPS: 100.02622903904029, Tokens per sec: 81734.00665200518, Loss: 2.275059223175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12047 , TFLOPS: 101.29295237134802, Tokens per sec: 82769.07889519332, Loss: 2.2369134426116943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12048 , TFLOPS: 98.7496924020519, Tokens per sec: 80690.91570494554, Loss: 2.2488083839416504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12049 , TFLOPS: 100.53091495172919, Tokens per sec: 82146.39850303445, Loss: 2.256929397583008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12050 , TFLOPS: 100.6215932819225, Tokens per sec: 82220.49410090329, Loss: 2.245811939239502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12051 , TFLOPS: 99.8898366150097, Tokens per sec: 81622.55689127644, Loss: 2.2589762210845947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12052 , TFLOPS: 100.56122316193716, Tokens per sec: 82171.16412179812, Loss: 2.2561614513397217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12053 , TFLOPS: 98.39721527577623, Tokens per sec: 80402.89757149725, Loss: 2.2361080646514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12054 , TFLOPS: 100.69288610286517, Tokens per sec: 82278.74930013603, Loss: 2.2466182708740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12055 , TFLOPS: 99.50122127484538, Tokens per sec: 81305.00929297935, Loss: 2.2465367317199707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12056 , TFLOPS: 100.75669921634315, Tokens per sec: 82330.89263785456, Loss: 2.254452705383301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12057 , TFLOPS: 99.33992879312265, Tokens per sec: 81173.21305412585, Loss: 2.238002061843872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12058 , TFLOPS: 100.6543753276193, Tokens per sec: 82247.2811543271, Loss: 2.2382164001464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12059 , TFLOPS: 98.79843531893579, Tokens per sec: 80730.74479708567, Loss: 2.2654247283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12060 , TFLOPS: 100.63914237779946, Tokens per sec: 82234.83391889826, Loss: 2.2609670162200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12061 , TFLOPS: 99.4312744213139, Tokens per sec: 81247.85391836659, Loss: 2.235045909881592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12062 , TFLOPS: 99.48549682782567, Tokens per sec: 81292.16044253627, Loss: 2.264216661453247 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12063 , TFLOPS: 100.69589326485642, Tokens per sec: 82281.20652961031, Loss: 2.2518553733825684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12064 , TFLOPS: 97.39726733209386, Tokens per sec: 79585.81436576434, Loss: 2.227118492126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12065 , TFLOPS: 100.79366555777337, Tokens per sec: 82361.09878703547, Loss: 2.2443392276763916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12066 , TFLOPS: 99.9281496582008, Tokens per sec: 81653.86346513336, Loss: 2.2872447967529297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12067 , TFLOPS: 99.20447524047214, Tokens per sec: 81062.53046937048, Loss: 2.2757911682128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12068 , TFLOPS: 99.51636231866968, Tokens per sec: 81317.38142965334, Loss: 2.238924026489258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12069 , TFLOPS: 100.12351793579809, Tokens per sec: 81813.50391398498, Loss: 2.2400591373443604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12070 , TFLOPS: 98.12893850667426, Tokens per sec: 80183.68171740562, Loss: 2.2391552925109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12071 , TFLOPS: 99.2639915392653, Tokens per sec: 81111.1627692002, Loss: 2.240638256072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12072 , TFLOPS: 98.83032702475748, Tokens per sec: 80756.80433087812, Loss: 2.2682931423187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12073 , TFLOPS: 100.05469560394175, Tokens per sec: 81757.26741498074, Loss: 2.268676280975342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12074 , TFLOPS: 99.33848416906136, Tokens per sec: 81172.03261461745, Loss: 2.257089853286743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12075 , TFLOPS: 100.11513543183808, Tokens per sec: 81806.65435421442, Loss: 2.2314043045043945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12076 , TFLOPS: 98.97586810579145, Tokens per sec: 80875.7296947517, Loss: 2.2478084564208984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12077 , TFLOPS: 101.20838619073237, Tokens per sec: 82699.97769208511, Loss: 2.2389369010925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12078 , TFLOPS: 98.33604668093295, Tokens per sec: 80352.9151380311, Loss: 2.236454725265503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12079 , TFLOPS: 97.66115032224836, Tokens per sec: 79801.43994997207, Loss: 2.2488842010498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12080 , TFLOPS: 100.23088651604299, Tokens per sec: 81901.23754481622, Loss: 2.255112409591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12081 , TFLOPS: 98.39076793217663, Tokens per sec: 80397.62928107263, Loss: 2.258941650390625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12082 , TFLOPS: 99.96468475907446, Tokens per sec: 81683.7172365544, Loss: 2.240342617034912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12083 , TFLOPS: 100.2331666332363, Tokens per sec: 81903.10068727031, Loss: 2.2593283653259277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12084 , TFLOPS: 100.80501228397583, Tokens per sec: 82370.37048910622, Loss: 2.2612826824188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12085 , TFLOPS: 100.00761980927412, Tokens per sec: 81718.80057131905, Loss: 2.2467727661132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12086 , TFLOPS: 100.09802924413053, Tokens per sec: 81792.67644788613, Loss: 2.269296169281006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12087 , TFLOPS: 100.04221761281009, Tokens per sec: 81747.07132721499, Loss: 2.268852472305298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12088 , TFLOPS: 100.72095172733141, Tokens per sec: 82301.68244436083, Loss: 2.249105453491211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12089 , TFLOPS: 99.23998652677601, Tokens per sec: 81091.5476555512, Loss: 2.2610058784484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12090 , TFLOPS: 101.2580853157772, Tokens per sec: 82740.58812652853, Loss: 2.2386937141418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12091 , TFLOPS: 98.13637228179029, Tokens per sec: 80189.75604641532, Loss: 2.2589468955993652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12092 , TFLOPS: 100.58888505045329, Tokens per sec: 82193.767362985, Loss: 2.272428035736084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12093 , TFLOPS: 99.39260202806317, Tokens per sec: 81216.25371032544, Loss: 2.2379090785980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12094 , TFLOPS: 100.62978407777842, Tokens per sec: 82227.18701104702, Loss: 2.258131265640259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12095 , TFLOPS: 98.87289475126917, Tokens per sec: 80791.58751600159, Loss: 2.2516286373138428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12096 , TFLOPS: 99.50848906983745, Tokens per sec: 81310.94799535707, Loss: 2.2712409496307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12097 , TFLOPS: 99.47654414593015, Tokens per sec: 81284.8449756969, Loss: 2.257477283477783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12098 , TFLOPS: 99.9305749725365, Tokens per sec: 81655.84525191021, Loss: 2.2422797679901123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12099 , TFLOPS: 99.55724372241298, Tokens per sec: 81350.78667703293, Loss: 2.2661776542663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12100 , TFLOPS: 99.51009837960555, Tokens per sec: 81312.26300379589, Loss: 2.242178201675415 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12101 , TFLOPS: 100.56561118038435, Tokens per sec: 82174.74968462897, Loss: 2.2567429542541504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12102 , TFLOPS: 98.35482473431858, Tokens per sec: 80368.2591688426, Loss: 2.2601547241210938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12103 , TFLOPS: 100.2448192201094, Tokens per sec: 81912.6223159686, Loss: 2.2400646209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12104 , TFLOPS: 100.64198058552196, Tokens per sec: 82237.1530914903, Loss: 2.2663795948028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12105 , TFLOPS: 98.79589870767444, Tokens per sec: 80728.67206672583, Loss: 2.2691681385040283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12106 , TFLOPS: 99.46133678947794, Tokens per sec: 81272.41865326761, Loss: 2.287759780883789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12107 , TFLOPS: 100.70160369925361, Tokens per sec: 82285.87267255591, Loss: 2.2567355632781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12108 , TFLOPS: 99.4918931695339, Tokens per sec: 81297.38705800254, Loss: 2.28180193901062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12109 , TFLOPS: 99.5090887543898, Tokens per sec: 81311.43801304203, Loss: 2.2559871673583984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12110 , TFLOPS: 99.41416580514333, Tokens per sec: 81233.87402767866, Loss: 2.267677068710327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12111 , TFLOPS: 100.1375037593739, Tokens per sec: 81824.93209045591, Loss: 2.2553913593292236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12112 , TFLOPS: 98.58499206450836, Tokens per sec: 80556.33481937484, Loss: 2.260761022567749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12113 , TFLOPS: 100.00371318789105, Tokens per sec: 81715.60837042157, Loss: 2.275942087173462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12114 , TFLOPS: 99.06326720792954, Tokens per sec: 80947.14575096178, Loss: 2.2651548385620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12115 , TFLOPS: 101.23959494704017, Tokens per sec: 82725.47917024909, Loss: 2.263831853866577 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12116 , TFLOPS: 97.59842501407097, Tokens per sec: 79750.18548596725, Loss: 2.2663841247558594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12117 , TFLOPS: 100.07645781872725, Tokens per sec: 81775.04988089122, Loss: 2.257293701171875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12118 , TFLOPS: 100.21311897911716, Tokens per sec: 81886.71923301705, Loss: 2.2283127307891846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12119 , TFLOPS: 98.37474668897777, Tokens per sec: 80384.53791083135, Loss: 2.237995147705078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12120 , TFLOPS: 99.99185853873841, Tokens per sec: 81705.92163143313, Loss: 2.2706785202026367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12121 , TFLOPS: 99.68250041901216, Tokens per sec: 81453.13714821835, Loss: 2.243177890777588 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12122 , TFLOPS: 100.75739561652287, Tokens per sec: 82331.46168436836, Loss: 2.2436819076538086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12123 , TFLOPS: 100.00550564867216, Tokens per sec: 81717.07303626786, Loss: 2.268009901046753 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12124 , TFLOPS: 99.52513264253385, Tokens per sec: 81324.54788705095, Loss: 2.2560043334960938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12125 , TFLOPS: 101.33188389532795, Tokens per sec: 82800.89084562392, Loss: 2.270508289337158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12126 , TFLOPS: 99.30855868429745, Tokens per sec: 81147.57973066636, Loss: 2.2442100048065186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12127 , TFLOPS: 100.06026980943388, Tokens per sec: 81761.82224178118, Loss: 2.2507238388061523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12128 , TFLOPS: 100.70561317428013, Tokens per sec: 82289.14891781316, Loss: 2.2463574409484863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12129 , TFLOPS: 99.33132424210085, Tokens per sec: 81166.18205398509, Loss: 2.28188157081604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12130 , TFLOPS: 99.41378566553772, Tokens per sec: 81233.56340582125, Loss: 2.2693872451782227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12131 , TFLOPS: 98.88871310092931, Tokens per sec: 80804.51309670942, Loss: 2.247281312942505 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12132 , TFLOPS: 99.98384274568004, Tokens per sec: 81699.37172058025, Loss: 2.268925189971924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12133 , TFLOPS: 99.87199218442943, Tokens per sec: 81607.9757476928, Loss: 2.2551090717315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12134 , TFLOPS: 98.96429196210897, Tokens per sec: 80866.2705297523, Loss: 2.246882438659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12135 , TFLOPS: 98.8895214243221, Tokens per sec: 80805.17359856179, Loss: 2.2587647438049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12136 , TFLOPS: 99.48721115622105, Tokens per sec: 81293.56126439887, Loss: 2.2748146057128906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12137 , TFLOPS: 99.01175578407245, Tokens per sec: 80905.05444050614, Loss: 2.252854824066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12138 , TFLOPS: 99.48900970274842, Tokens per sec: 81295.0309030651, Loss: 2.279226064682007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12139 , TFLOPS: 100.73895972340905, Tokens per sec: 82316.39724152294, Loss: 2.2652926445007324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12140 , TFLOPS: 98.36461848605225, Tokens per sec: 80376.26189549777, Loss: 2.2589330673217773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12141 , TFLOPS: 100.25818457445384, Tokens per sec: 81923.54349105827, Loss: 2.237224817276001 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12142 , TFLOPS: 100.6803410227114, Tokens per sec: 82268.49839220328, Loss: 2.2540299892425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12143 , TFLOPS: 99.56142646382813, Tokens per sec: 81354.2045027166, Loss: 2.249756336212158 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12144 , TFLOPS: 100.00126530111737, Tokens per sec: 81713.60813912458, Loss: 2.2506558895111084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12145 , TFLOPS: 100.19051531391551, Tokens per sec: 81868.24919631067, Loss: 2.2611546516418457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12146 , TFLOPS: 99.3410116387475, Tokens per sec: 81174.09787516085, Loss: 2.2423346042633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12147 , TFLOPS: 99.35095410969721, Tokens per sec: 81182.22212411584, Loss: 2.27203369140625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12148 , TFLOPS: 100.07679245568058, Tokens per sec: 81775.32332136009, Loss: 2.2643587589263916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12149 , TFLOPS: 100.18638867777432, Tokens per sec: 81864.87721569063, Loss: 2.2699050903320312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12150 , TFLOPS: 99.48234245708431, Tokens per sec: 81289.58292500279, Loss: 2.2608461380004883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12151 , TFLOPS: 98.81333231248851, Tokens per sec: 80742.91751400034, Loss: 2.2744836807250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12152 , TFLOPS: 99.55752576432205, Tokens per sec: 81351.01714073724, Loss: 2.2381906509399414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12153 , TFLOPS: 101.2433688974088, Tokens per sec: 82728.56296224541, Loss: 2.2478106021881104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12154 , TFLOPS: 98.8212711271724, Tokens per sec: 80749.4045238418, Loss: 2.273336887359619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12155 , TFLOPS: 100.02850646610372, Tokens per sec: 81735.8675962849, Loss: 2.2557413578033447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12156 , TFLOPS: 100.07030406777773, Tokens per sec: 81770.02149257873, Loss: 2.250296115875244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12157 , TFLOPS: 99.69226313522323, Tokens per sec: 81461.11451494908, Loss: 2.249873161315918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12158 , TFLOPS: 99.9728022297212, Tokens per sec: 81690.35023078213, Loss: 2.276054859161377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12159 , TFLOPS: 99.03957306304288, Tokens per sec: 80927.78465523334, Loss: 2.265540361404419 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12160 , TFLOPS: 100.69309443154428, Tokens per sec: 82278.91953086258, Loss: 2.254486560821533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12161 , TFLOPS: 99.37722001788471, Tokens per sec: 81203.68467384046, Loss: 2.2533185482025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12162 , TFLOPS: 99.97797412548792, Tokens per sec: 81694.57631994956, Loss: 2.2555997371673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12163 , TFLOPS: 100.6990961964141, Tokens per sec: 82283.82372743689, Loss: 2.2634763717651367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12164 , TFLOPS: 99.93552311168904, Tokens per sec: 81659.88850378817, Loss: 2.2532646656036377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12165 , TFLOPS: 99.28369983587925, Tokens per sec: 81127.26692570013, Loss: 2.248255968093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12166 , TFLOPS: 100.61334580318376, Tokens per sec: 82213.75487371688, Loss: 2.2654192447662354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12167 , TFLOPS: 99.3947810785576, Tokens per sec: 81218.0342685782, Loss: 2.262871265411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12168 , TFLOPS: 99.4124446251397, Tokens per sec: 81232.46760719057, Loss: 2.2569010257720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12169 , TFLOPS: 98.92644728037851, Tokens per sec: 80835.34666610157, Loss: 2.2768166065216064 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12170 , TFLOPS: 99.99933950096465, Tokens per sec: 81712.03451824529, Loss: 2.2744719982147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12171 , TFLOPS: 100.16236163033616, Tokens per sec: 81845.24409671771, Loss: 2.2788517475128174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12172 , TFLOPS: 98.47279283198137, Tokens per sec: 80464.65393821185, Loss: 2.247138023376465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12173 , TFLOPS: 99.45666638118738, Tokens per sec: 81268.6023424264, Loss: 2.237356424331665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12174 , TFLOPS: 100.00545982358801, Tokens per sec: 81717.03559141196, Loss: 2.2567481994628906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12175 , TFLOPS: 98.91123996976634, Tokens per sec: 80822.92038112936, Loss: 2.2595090866088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12176 , TFLOPS: 99.97425806008977, Tokens per sec: 81691.5398272527, Loss: 2.263932228088379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12177 , TFLOPS: 100.65826214901614, Tokens per sec: 82250.45717614611, Loss: 2.2698891162872314 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12178 , TFLOPS: 98.95942216059375, Tokens per sec: 80862.29128957436, Loss: 2.2615339756011963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12179 , TFLOPS: 100.1081979186747, Tokens per sec: 81800.98553362115, Loss: 2.248540163040161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12180 , TFLOPS: 100.7408489309585, Tokens per sec: 82317.94096164413, Loss: 2.2540180683135986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12181 , TFLOPS: 99.53161572194249, Tokens per sec: 81329.84537812507, Loss: 2.2490813732147217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12182 , TFLOPS: 99.32014086668384, Tokens per sec: 81157.04383004617, Loss: 2.270623207092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12183 , TFLOPS: 99.95157963217592, Tokens per sec: 81673.00870000962, Loss: 2.2493739128112793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12184 , TFLOPS: 100.02072110663774, Tokens per sec: 81729.50597865226, Loss: 2.268890142440796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12185 , TFLOPS: 99.69839047340244, Tokens per sec: 81466.12132070703, Loss: 2.264033794403076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12186 , TFLOPS: 99.97988936600979, Tokens per sec: 81696.14131228265, Loss: 2.2734553813934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12187 , TFLOPS: 99.51541572537997, Tokens per sec: 81316.60794390884, Loss: 2.2637665271759033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12188 , TFLOPS: 98.89625944287748, Tokens per sec: 80810.67940697541, Loss: 2.251774549484253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12189 , TFLOPS: 100.01056665642041, Tokens per sec: 81721.20851598088, Loss: 2.2606523036956787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12190 , TFLOPS: 99.52111807039411, Tokens per sec: 81321.2674768113, Loss: 2.24543833732605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12191 , TFLOPS: 100.79278350824076, Tokens per sec: 82360.37804165641, Loss: 2.2412734031677246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12192 , TFLOPS: 98.25086888277181, Tokens per sec: 80283.31416648204, Loss: 2.2715604305267334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12193 , TFLOPS: 101.2534900879679, Tokens per sec: 82736.83324759388, Loss: 2.2528738975524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12194 , TFLOPS: 98.97034255294888, Tokens per sec: 80871.21462328365, Loss: 2.258937358856201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12195 , TFLOPS: 100.36000490184625, Tokens per sec: 82006.74350165902, Loss: 2.2526252269744873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12196 , TFLOPS: 99.05762382089758, Tokens per sec: 80942.53439414426, Loss: 2.2665984630584717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12197 , TFLOPS: 98.82252791257935, Tokens per sec: 80750.43147555036, Loss: 2.2530176639556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12198 , TFLOPS: 100.02136229073234, Tokens per sec: 81730.02990668152, Loss: 2.2516767978668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12199 , TFLOPS: 100.56714831820496, Tokens per sec: 82176.00571951164, Loss: 2.259194850921631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12200 , TFLOPS: 98.88675025884102, Tokens per sec: 80802.90920791104, Loss: 2.2753868103027344 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/model.pt +[rank0]:[2024-08-29 10:07:12,770] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007691836995945778, 'preprocessing_with_comm': 0.0016498450058861636, 'state_converting': 2.400254578999011, : 2.4112690280016977}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012200, took 14.52s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010200 + eval ppl=7.922023296356201, eval loss=2.0696465969085693 +------------------------------------------------------------------ +iteration: 12201 , TFLOPS: 100.41352569555995, Tokens per sec: 82050.4767199503, Loss: 2.270134449005127 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12202 , TFLOPS: 98.01745721634992, Tokens per sec: 80092.58748529787, Loss: 2.2677507400512695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12203 , TFLOPS: 100.15063899276761, Tokens per sec: 81835.6652277929, Loss: 2.2454066276550293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12204 , TFLOPS: 98.99032622662398, Tokens per sec: 80887.5437974691, Loss: 2.2337870597839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12205 , TFLOPS: 99.61862685770521, Tokens per sec: 81400.94441702269, Loss: 2.2635765075683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12206 , TFLOPS: 100.61390656130614, Tokens per sec: 82214.21308361387, Loss: 2.253824234008789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12207 , TFLOPS: 98.520411985391, Tokens per sec: 80503.56477428891, Loss: 2.282907485961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12208 , TFLOPS: 100.01391384848806, Tokens per sec: 81723.94359278375, Loss: 2.2598071098327637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12209 , TFLOPS: 100.56436828951345, Tokens per sec: 82173.73408650351, Loss: 2.2693142890930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12210 , TFLOPS: 100.55781213077118, Tokens per sec: 82168.37688042465, Loss: 2.261798858642578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12211 , TFLOPS: 99.98947424105125, Tokens per sec: 81703.97336041566, Loss: 2.2665858268737793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12212 , TFLOPS: 101.3014911850406, Tokens per sec: 82776.05617967015, Loss: 2.260169506072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12213 , TFLOPS: 100.06907683859481, Tokens per sec: 81769.01869202167, Loss: 2.2480907440185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12214 , TFLOPS: 100.25544439968023, Tokens per sec: 81921.30442371238, Loss: 2.2688257694244385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12215 , TFLOPS: 99.78750391328185, Tokens per sec: 81538.93820641653, Loss: 2.252444267272949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12216 , TFLOPS: 99.97966767332647, Tokens per sec: 81695.96016148424, Loss: 2.257821559906006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12217 , TFLOPS: 98.73074764422384, Tokens per sec: 80675.43545564279, Loss: 2.267404556274414 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12218 , TFLOPS: 100.74821970939664, Tokens per sec: 82323.96381444664, Loss: 2.2340219020843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12219 , TFLOPS: 100.65468624737662, Tokens per sec: 82247.53521486456, Loss: 2.2593843936920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12220 , TFLOPS: 99.38622190385709, Tokens per sec: 81211.04034659766, Loss: 2.2608301639556885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12221 , TFLOPS: 100.66795271671985, Tokens per sec: 82258.37558847423, Loss: 2.269514560699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12222 , TFLOPS: 99.81996658549525, Tokens per sec: 81565.46429154562, Loss: 2.2708945274353027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12223 , TFLOPS: 100.12903583470033, Tokens per sec: 81818.01273122137, Loss: 2.263866424560547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12224 , TFLOPS: 99.40115449882406, Tokens per sec: 81223.24215434435, Loss: 2.2656102180480957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12225 , TFLOPS: 99.96632097025669, Tokens per sec: 81685.05422683121, Loss: 2.270237445831299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12226 , TFLOPS: 99.51245274649318, Tokens per sec: 81314.18681758657, Loss: 2.2503821849823 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12227 , TFLOPS: 100.47973568053952, Tokens per sec: 82104.57860307374, Loss: 2.2610106468200684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12228 , TFLOPS: 98.78335153384931, Tokens per sec: 80718.41944800034, Loss: 2.2565414905548096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12229 , TFLOPS: 99.95732045554385, Tokens per sec: 81677.6996745655, Loss: 2.2445034980773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12230 , TFLOPS: 98.9536860345994, Tokens per sec: 80857.60415336337, Loss: 2.2461090087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12231 , TFLOPS: 99.31673198193117, Tokens per sec: 81154.25834256226, Loss: 2.2702581882476807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12232 , TFLOPS: 100.1412394093449, Tokens per sec: 81827.9845862114, Loss: 2.2604713439941406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12233 , TFLOPS: 98.74086080324962, Tokens per sec: 80683.69917821862, Loss: 2.2529373168945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12234 , TFLOPS: 98.9232813072408, Tokens per sec: 80832.75966795135, Loss: 2.2556257247924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12235 , TFLOPS: 99.01698793976107, Tokens per sec: 80909.32976960698, Loss: 2.2673885822296143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12236 , TFLOPS: 99.04609483185256, Tokens per sec: 80933.11376041299, Loss: 2.2713301181793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12237 , TFLOPS: 98.45400839578737, Tokens per sec: 80449.30469184332, Loss: 2.238537311553955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12238 , TFLOPS: 98.45512407188184, Tokens per sec: 80450.21633950018, Loss: 2.266232967376709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12239 , TFLOPS: 100.16806930792819, Tokens per sec: 81849.90798700684, Loss: 2.2831356525421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12240 , TFLOPS: 98.90470226876315, Tokens per sec: 80817.57825734426, Loss: 2.241199493408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12241 , TFLOPS: 99.41247316642283, Tokens per sec: 81232.49092900773, Loss: 2.242669105529785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12242 , TFLOPS: 99.41950505599281, Tokens per sec: 81238.23686699232, Loss: 2.249293327331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12243 , TFLOPS: 99.57800272486011, Tokens per sec: 81367.74939231675, Loss: 2.2537269592285156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12244 , TFLOPS: 100.68791901643672, Tokens per sec: 82274.69056595115, Loss: 2.2727341651916504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12245 , TFLOPS: 98.95587454606552, Tokens per sec: 80859.39244241962, Loss: 2.2508509159088135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12246 , TFLOPS: 100.22204046756114, Tokens per sec: 81894.00921088418, Loss: 2.265606641769409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12247 , TFLOPS: 100.08712101207263, Tokens per sec: 81783.76305066874, Loss: 2.2496700286865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12248 , TFLOPS: 100.69701211926886, Tokens per sec: 82282.12077435259, Loss: 2.2488515377044678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12249 , TFLOPS: 100.6362583729357, Tokens per sec: 82232.47732428322, Loss: 2.276634693145752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12250 , TFLOPS: 101.39175615406253, Tokens per sec: 82849.81400948498, Loss: 2.2579915523529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12251 , TFLOPS: 99.55553448587915, Tokens per sec: 81349.39001586149, Loss: 2.2581844329833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12252 , TFLOPS: 100.54246445016548, Tokens per sec: 82155.83589551729, Loss: 2.247262954711914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12253 , TFLOPS: 100.705204701137, Tokens per sec: 82288.81514389288, Loss: 2.2454261779785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12254 , TFLOPS: 99.49028189013427, Tokens per sec: 81296.07044012708, Loss: 2.269566774368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12255 , TFLOPS: 99.51274101675241, Tokens per sec: 81314.42237063612, Loss: 2.262661933898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12256 , TFLOPS: 100.70376261591214, Tokens per sec: 82287.63677893305, Loss: 2.2518391609191895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12257 , TFLOPS: 100.65763455640212, Tokens per sec: 82249.94435406552, Loss: 2.2299928665161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12258 , TFLOPS: 98.8445614112189, Tokens per sec: 80768.43561448145, Loss: 2.2507827281951904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12259 , TFLOPS: 100.67084286845152, Tokens per sec: 82260.7372058533, Loss: 2.262350559234619 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12260 , TFLOPS: 100.05018323713543, Tokens per sec: 81753.58024390464, Loss: 2.257164478302002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12261 , TFLOPS: 100.58054127194315, Tokens per sec: 82186.94944677642, Loss: 2.2487568855285645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12262 , TFLOPS: 98.91960948637812, Tokens per sec: 80829.75932860338, Loss: 2.243027687072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12263 , TFLOPS: 100.66498104502908, Tokens per sec: 82255.94735903807, Loss: 2.260267972946167 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12264 , TFLOPS: 98.7941343732438, Tokens per sec: 80727.23038364456, Loss: 2.2799339294433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12265 , TFLOPS: 100.7235527337535, Tokens per sec: 82303.80779366418, Loss: 2.2536299228668213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12266 , TFLOPS: 98.83026034027363, Tokens per sec: 80756.74984126975, Loss: 2.2583398818969727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12267 , TFLOPS: 99.9778759518851, Tokens per sec: 81694.49609977148, Loss: 2.2543458938598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12268 , TFLOPS: 99.09241893154582, Tokens per sec: 80970.96637476077, Loss: 2.257124900817871 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12269 , TFLOPS: 99.44334951890596, Tokens per sec: 81257.72079144971, Loss: 2.2821004390716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12270 , TFLOPS: 99.87389504190645, Tokens per sec: 81609.53062152117, Loss: 2.2489471435546875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12271 , TFLOPS: 98.38944136895057, Tokens per sec: 80396.54531211183, Loss: 2.263646125793457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12272 , TFLOPS: 100.20848730558332, Tokens per sec: 81882.93457334266, Loss: 2.230506181716919 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12273 , TFLOPS: 97.70490544350996, Tokens per sec: 79837.19338591195, Loss: 2.266406536102295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12274 , TFLOPS: 100.64689921764392, Tokens per sec: 82241.172232414, Loss: 2.2733123302459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12275 , TFLOPS: 99.04057801555321, Tokens per sec: 80928.6058277993, Loss: 2.2629122734069824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12276 , TFLOPS: 99.06782885155909, Tokens per sec: 80950.87318739864, Loss: 2.2631540298461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12277 , TFLOPS: 99.94258793038769, Tokens per sec: 81665.66134901147, Loss: 2.2598791122436523 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12278 , TFLOPS: 99.44322027970662, Tokens per sec: 81257.61518677302, Loss: 2.266247510910034 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12279 , TFLOPS: 98.65039691455621, Tokens per sec: 80609.77880601957, Loss: 2.2633633613586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12280 , TFLOPS: 99.56259755692034, Tokens per sec: 81355.16143302886, Loss: 2.2354958057403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12281 , TFLOPS: 99.20761140761975, Tokens per sec: 81065.09311227892, Loss: 2.2738821506500244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12282 , TFLOPS: 100.73895115915094, Tokens per sec: 82316.39024344717, Loss: 2.267615795135498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12283 , TFLOPS: 98.38580499509037, Tokens per sec: 80393.57393742209, Loss: 2.2606730461120605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12284 , TFLOPS: 100.26203333812037, Tokens per sec: 81926.6884149262, Loss: 2.2454538345336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12285 , TFLOPS: 99.32348582233946, Tokens per sec: 81159.7770794192, Loss: 2.2742743492126465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12286 , TFLOPS: 100.73493266485441, Tokens per sec: 82313.10662831223, Loss: 2.2485687732696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12287 , TFLOPS: 100.68308705460441, Tokens per sec: 82270.74224555222, Loss: 2.240002393722534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12288 , TFLOPS: 101.28722524790847, Tokens per sec: 82764.39911520158, Loss: 2.2621960639953613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12289 , TFLOPS: 99.31942890342444, Tokens per sec: 81156.46206653933, Loss: 2.2429938316345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12290 , TFLOPS: 100.04629553839415, Tokens per sec: 81750.40350518486, Loss: 2.2551615238189697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12291 , TFLOPS: 100.65757253902895, Tokens per sec: 82249.89367807344, Loss: 2.2420616149902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12292 , TFLOPS: 100.05981234904375, Tokens per sec: 81761.44843912033, Loss: 2.2567410469055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12293 , TFLOPS: 99.00734067731203, Tokens per sec: 80901.44674311724, Loss: 2.277359962463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12294 , TFLOPS: 99.44517499688995, Tokens per sec: 81259.21243650242, Loss: 2.280583143234253 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12295 , TFLOPS: 101.32771509523252, Tokens per sec: 82797.48441175147, Loss: 2.2666990756988525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12296 , TFLOPS: 97.99325718449583, Tokens per sec: 80072.81301630566, Loss: 2.262629747390747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12297 , TFLOPS: 100.70511119551571, Tokens per sec: 82288.73873804267, Loss: 2.2667617797851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12298 , TFLOPS: 100.69491515115801, Tokens per sec: 82280.40728772846, Loss: 2.2512176036834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12299 , TFLOPS: 99.41582711436374, Tokens per sec: 81235.23152620855, Loss: 2.2554423809051514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12300 , TFLOPS: 98.73420620504189, Tokens per sec: 80678.26153471848, Loss: 2.2859764099121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12301 , TFLOPS: 100.23564079191101, Tokens per sec: 81905.12238601394, Loss: 2.267108917236328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12302 , TFLOPS: 100.07717614198037, Tokens per sec: 81775.63684131262, Loss: 2.240795135498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12303 , TFLOPS: 99.49696334515822, Tokens per sec: 81301.53002902311, Loss: 2.23759388923645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12304 , TFLOPS: 99.45327131775252, Tokens per sec: 81265.82814869708, Loss: 2.2389075756073 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12305 , TFLOPS: 99.3408983753082, Tokens per sec: 81174.00532468893, Loss: 2.2519216537475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12306 , TFLOPS: 99.63770804155675, Tokens per sec: 81416.53614354116, Loss: 2.2840535640716553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12307 , TFLOPS: 100.07132988466842, Tokens per sec: 81770.85971396699, Loss: 2.2433857917785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12308 , TFLOPS: 98.74407983861644, Tokens per sec: 80686.32953488218, Loss: 2.2450687885284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12309 , TFLOPS: 99.02855789697362, Tokens per sec: 80918.78387948255, Loss: 2.2695813179016113 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12310 , TFLOPS: 99.94569732920046, Tokens per sec: 81668.20211882441, Loss: 2.272045135498047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12311 , TFLOPS: 99.63163209731545, Tokens per sec: 81411.57133309287, Loss: 2.2570369243621826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12312 , TFLOPS: 99.45489588771186, Tokens per sec: 81267.15562663099, Loss: 2.2398452758789062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12313 , TFLOPS: 99.32741457684887, Tokens per sec: 81162.98736586397, Loss: 2.2625083923339844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12314 , TFLOPS: 99.48148678394759, Tokens per sec: 81288.88373245578, Loss: 2.2602391242980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12315 , TFLOPS: 99.32531482849458, Tokens per sec: 81161.27160743135, Loss: 2.2645084857940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12316 , TFLOPS: 100.08756691176039, Tokens per sec: 81784.12740678209, Loss: 2.2515206336975098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12317 , TFLOPS: 99.40956807582677, Tokens per sec: 81230.11710469802, Loss: 2.269552230834961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12318 , TFLOPS: 98.87805934472473, Tokens per sec: 80795.80763826259, Loss: 2.2446694374084473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12319 , TFLOPS: 99.38261233072735, Tokens per sec: 81208.09087147466, Loss: 2.2259225845336914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12320 , TFLOPS: 100.75817684785376, Tokens per sec: 82332.1000485995, Loss: 2.2521023750305176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12321 , TFLOPS: 95.8975805577016, Tokens per sec: 78360.38169703641, Loss: 2.2610511779785156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12322 , TFLOPS: 100.72448748909858, Tokens per sec: 82304.5716063194, Loss: 2.249617338180542 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12323 , TFLOPS: 99.47137428472806, Tokens per sec: 81280.6205490246, Loss: 2.272429943084717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12324 , TFLOPS: 99.45974763800325, Tokens per sec: 81271.12011668927, Loss: 2.260774612426758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12325 , TFLOPS: 100.6594588593407, Tokens per sec: 82251.43503895839, Loss: 2.240539312362671 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12326 , TFLOPS: 99.63173552718976, Tokens per sec: 81411.65584830569, Loss: 2.2509326934814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12327 , TFLOPS: 100.70702451334799, Tokens per sec: 82290.30215929661, Loss: 2.245274782180786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12328 , TFLOPS: 98.27050951414996, Tokens per sec: 80299.36303197595, Loss: 2.257075309753418 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12329 , TFLOPS: 101.29550237546505, Tokens per sec: 82771.16256920029, Loss: 2.2573094367980957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12330 , TFLOPS: 99.63555793968804, Tokens per sec: 81414.7792399557, Loss: 2.2532992362976074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12331 , TFLOPS: 98.99236320493024, Tokens per sec: 80889.20826487963, Loss: 2.2519352436065674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12332 , TFLOPS: 101.30020341294191, Tokens per sec: 82775.00390793811, Loss: 2.2639167308807373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12333 , TFLOPS: 100.85473492601896, Tokens per sec: 82411.00013989434, Loss: 2.2482213973999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12334 , TFLOPS: 98.84661209283543, Tokens per sec: 80770.11127921956, Loss: 2.244858741760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12335 , TFLOPS: 99.29238683541081, Tokens per sec: 81134.36529664064, Loss: 2.2602131366729736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12336 , TFLOPS: 100.19532922286857, Tokens per sec: 81872.18276523726, Loss: 2.249474048614502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12337 , TFLOPS: 100.04035952235479, Tokens per sec: 81745.55303367246, Loss: 2.2578582763671875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12338 , TFLOPS: 99.43289183180914, Tokens per sec: 81249.1755461181, Loss: 2.2740554809570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12339 , TFLOPS: 100.17185437265498, Tokens per sec: 81853.00086083132, Loss: 2.2494447231292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12340 , TFLOPS: 100.08493649166061, Tokens per sec: 81781.97802280553, Loss: 2.2369656562805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12341 , TFLOPS: 98.81839517080563, Tokens per sec: 80747.05450586081, Loss: 2.250527858734131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12342 , TFLOPS: 100.08579048027185, Tokens per sec: 81782.6758388834, Loss: 2.2607779502868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12343 , TFLOPS: 98.87999782535782, Tokens per sec: 80797.39162068881, Loss: 2.2476553916931152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12344 , TFLOPS: 99.52747684428041, Tokens per sec: 81326.4633946432, Loss: 2.2577788829803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12345 , TFLOPS: 100.13264231954926, Tokens per sec: 81820.9596828306, Loss: 2.2572686672210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12346 , TFLOPS: 98.43578016543644, Tokens per sec: 80434.40993558719, Loss: 2.257927179336548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12347 , TFLOPS: 99.42570244842989, Tokens per sec: 81243.30091588771, Loss: 2.253687858581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12348 , TFLOPS: 99.0985181731857, Tokens per sec: 80975.95022211316, Loss: 2.249758243560791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12349 , TFLOPS: 99.06266918090266, Tokens per sec: 80946.65708768353, Loss: 2.262695550918579 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12350 , TFLOPS: 100.66092507012041, Tokens per sec: 82252.63311753012, Loss: 2.264207363128662 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12351 , TFLOPS: 98.94557740958346, Tokens per sec: 80850.97838712827, Loss: 2.2534255981445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12352 , TFLOPS: 101.2698857555785, Tokens per sec: 82750.23056966053, Loss: 2.265113115310669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12353 , TFLOPS: 98.21952088677621, Tokens per sec: 80257.69891198483, Loss: 2.260707378387451 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12354 , TFLOPS: 100.74458321561595, Tokens per sec: 82320.99234176676, Loss: 2.245258092880249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12355 , TFLOPS: 99.61036336403612, Tokens per sec: 81394.19210362442, Loss: 2.252657890319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12356 , TFLOPS: 98.83822193496188, Tokens per sec: 80763.2554652392, Loss: 2.2722268104553223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12357 , TFLOPS: 100.71503175847845, Tokens per sec: 82296.84508541753, Loss: 2.260308265686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12358 , TFLOPS: 99.47151207044236, Tokens per sec: 81280.73313727869, Loss: 2.267594337463379 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12359 , TFLOPS: 100.04798779831295, Tokens per sec: 81751.78629432716, Loss: 2.262394428253174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12360 , TFLOPS: 99.4226486649754, Tokens per sec: 81240.80559081567, Loss: 2.259514808654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12361 , TFLOPS: 100.57562210517422, Tokens per sec: 82182.9298689789, Loss: 2.2562453746795654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12362 , TFLOPS: 99.24084094567792, Tokens per sec: 81092.24582323065, Loss: 2.2661232948303223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12363 , TFLOPS: 100.67804002946242, Tokens per sec: 82266.61819138672, Loss: 2.267549753189087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12364 , TFLOPS: 100.23743081951024, Tokens per sec: 81906.58506364468, Loss: 2.267831325531006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12365 , TFLOPS: 100.6698198721195, Tokens per sec: 82259.90128921614, Loss: 2.2385408878326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12366 , TFLOPS: 97.1333862353407, Tokens per sec: 79370.19032870358, Loss: 2.2685585021972656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12367 , TFLOPS: 101.23945753059012, Tokens per sec: 82725.36688373033, Loss: 2.265420436859131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12368 , TFLOPS: 99.48151714005344, Tokens per sec: 81288.90853721129, Loss: 2.257598400115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12369 , TFLOPS: 99.01610015412935, Tokens per sec: 80908.60433711366, Loss: 2.2492761611938477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12370 , TFLOPS: 100.70238503826515, Tokens per sec: 82286.5111247757, Loss: 2.2566189765930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12371 , TFLOPS: 100.09354814140234, Tokens per sec: 81789.01482349307, Loss: 2.2583189010620117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12372 , TFLOPS: 98.82448282901116, Tokens per sec: 80752.02888809082, Loss: 2.2695705890655518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12373 , TFLOPS: 99.41371884993907, Tokens per sec: 81233.5088090756, Loss: 2.2684786319732666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12374 , TFLOPS: 101.19465808031617, Tokens per sec: 82688.76009967105, Loss: 2.2563681602478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12375 , TFLOPS: 99.52040370400496, Tokens per sec: 81320.68374964531, Loss: 2.2475101947784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12376 , TFLOPS: 98.89197358420687, Tokens per sec: 80807.17732152782, Loss: 2.273430824279785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12377 , TFLOPS: 99.27276197113791, Tokens per sec: 81118.32931485432, Loss: 2.2558906078338623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12378 , TFLOPS: 100.11705808477042, Tokens per sec: 81808.22540341892, Loss: 2.2641379833221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12379 , TFLOPS: 98.96724825201453, Tokens per sec: 80868.68619033581, Loss: 2.2824039459228516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12380 , TFLOPS: 100.10189861634449, Tokens per sec: 81795.83821152865, Loss: 2.2410449981689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12381 , TFLOPS: 98.82069753671753, Tokens per sec: 80748.9358283156, Loss: 2.2506279945373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12382 , TFLOPS: 100.00470409853212, Tokens per sec: 81716.41806901469, Loss: 2.2525832653045654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12383 , TFLOPS: 98.7589259287717, Tokens per sec: 80698.46065732041, Loss: 2.240440845489502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12384 , TFLOPS: 99.52488716914517, Tokens per sec: 81324.34730442603, Loss: 2.2619664669036865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12385 , TFLOPS: 98.88479648947616, Tokens per sec: 80801.31273266852, Loss: 2.262432098388672 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12386 , TFLOPS: 100.22085817645544, Tokens per sec: 81893.04313038681, Loss: 2.2603416442871094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12387 , TFLOPS: 99.04619516161912, Tokens per sec: 80933.19574244796, Loss: 2.2565112113952637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12388 , TFLOPS: 100.11467849961112, Tokens per sec: 81806.28098312927, Loss: 2.270728588104248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12389 , TFLOPS: 99.35915961491038, Tokens per sec: 81188.927053654, Loss: 2.2536070346832275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12390 , TFLOPS: 100.68073516526277, Tokens per sec: 82268.82045622815, Loss: 2.2580223083496094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12391 , TFLOPS: 99.2760310164725, Tokens per sec: 81121.00053594992, Loss: 2.2487499713897705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12392 , TFLOPS: 100.04928447705949, Tokens per sec: 81752.84584391044, Loss: 2.267021417617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12393 , TFLOPS: 100.65880231214483, Tokens per sec: 82250.89855734361, Loss: 2.247769355773926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12394 , TFLOPS: 98.83349732596355, Tokens per sec: 80759.39486560441, Loss: 2.27243709564209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12395 , TFLOPS: 99.89529061363444, Tokens per sec: 81627.01349395113, Loss: 2.261096239089966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12396 , TFLOPS: 100.14555788987684, Tokens per sec: 81831.51332782162, Loss: 2.2838046550750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12397 , TFLOPS: 99.81360592538904, Tokens per sec: 81560.26683243488, Loss: 2.247979164123535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12398 , TFLOPS: 100.11366269908838, Tokens per sec: 81805.45094637312, Loss: 2.253925323486328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12399 , TFLOPS: 100.57731824484264, Tokens per sec: 82184.3158283645, Loss: 2.255814790725708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12400 , TFLOPS: 98.78535220338978, Tokens per sec: 80720.05424658368, Loss: 2.2682414054870605 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/optimizer.pt +[rank0]:[2024-08-29 11:01:13,273] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0076866969975526445, 'preprocessing_with_comm': 0.0017936029980774038, 'state_converting': 2.3978095979982754, : 2.408904789997905}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012400, took 14.55s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010400 + eval ppl=7.951868534088135, eval loss=2.073406934738159 +------------------------------------------------------------------ +iteration: 12401 , TFLOPS: 98.77615723700794, Tokens per sec: 80712.54080285357, Loss: 2.259455919265747 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12402 , TFLOPS: 99.03689742105834, Tokens per sec: 80925.59831929061, Loss: 2.2548468112945557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12403 , TFLOPS: 98.56556660844642, Tokens per sec: 80540.46177917092, Loss: 2.2413527965545654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12404 , TFLOPS: 100.8068295470792, Tokens per sec: 82371.85542156851, Loss: 2.2784039974212646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12405 , TFLOPS: 98.43295531429656, Tokens per sec: 80432.10167700266, Loss: 2.2417941093444824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12406 , TFLOPS: 98.4840894314778, Tokens per sec: 80473.88468046072, Loss: 2.2510464191436768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12407 , TFLOPS: 100.63911508323537, Tokens per sec: 82234.81161580732, Loss: 2.2670109272003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12408 , TFLOPS: 100.67369586785901, Tokens per sec: 82263.0684651119, Loss: 2.2676022052764893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12409 , TFLOPS: 100.50276251101909, Tokens per sec: 82123.39441902195, Loss: 2.237422466278076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12410 , TFLOPS: 101.29732754380021, Tokens per sec: 82772.65396123097, Loss: 2.2535531520843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12411 , TFLOPS: 100.11827438003645, Tokens per sec: 81809.21926959102, Loss: 2.262998580932617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12412 , TFLOPS: 100.68563324666923, Tokens per sec: 82272.8228046332, Loss: 2.247541904449463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12413 , TFLOPS: 98.65612508835734, Tokens per sec: 80614.45944428879, Loss: 2.2517967224121094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12414 , TFLOPS: 99.25411792394503, Tokens per sec: 81103.09478395253, Loss: 2.242340087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12415 , TFLOPS: 100.54650345443025, Tokens per sec: 82159.13626987525, Loss: 2.263340473175049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12416 , TFLOPS: 100.37497878013245, Tokens per sec: 82018.9790430686, Loss: 2.2749111652374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12417 , TFLOPS: 99.29403535142535, Tokens per sec: 81135.7123415127, Loss: 2.2804486751556396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12418 , TFLOPS: 101.23236310403759, Tokens per sec: 82719.56984516748, Loss: 2.2776637077331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12419 , TFLOPS: 99.91489273283858, Tokens per sec: 81643.03089015627, Loss: 2.2612359523773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12420 , TFLOPS: 99.45341262625068, Tokens per sec: 81265.94361550851, Loss: 2.270155191421509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12421 , TFLOPS: 100.64796463091322, Tokens per sec: 82242.04280902253, Loss: 2.2487292289733887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12422 , TFLOPS: 99.989090218211, Tokens per sec: 81703.65956546727, Loss: 2.2614102363586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12423 , TFLOPS: 99.43318189507792, Tokens per sec: 81249.41256428191, Loss: 2.2501373291015625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12424 , TFLOPS: 98.85975312386499, Tokens per sec: 80780.84915395404, Loss: 2.2557380199432373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12425 , TFLOPS: 101.38013229480264, Tokens per sec: 82840.31585485897, Loss: 2.2747859954833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12426 , TFLOPS: 99.39247450192983, Tokens per sec: 81216.14950543907, Loss: 2.2526731491088867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12427 , TFLOPS: 99.0067165488829, Tokens per sec: 80900.93675171131, Loss: 2.2591230869293213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12428 , TFLOPS: 100.09845832343797, Tokens per sec: 81793.02705963372, Loss: 2.236629009246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12429 , TFLOPS: 100.14442783355072, Tokens per sec: 81830.58992970732, Loss: 2.2324416637420654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12430 , TFLOPS: 97.7322131258001, Tokens per sec: 79859.5071960735, Loss: 2.259023666381836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12431 , TFLOPS: 100.63161647866995, Tokens per sec: 82228.68431298567, Loss: 2.2678768634796143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12432 , TFLOPS: 98.80028833122087, Tokens per sec: 80732.25894112459, Loss: 2.250959873199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12433 , TFLOPS: 99.12603279450693, Tokens per sec: 80998.4331274841, Loss: 2.269092321395874 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12434 , TFLOPS: 99.02000199638846, Tokens per sec: 80911.79263286594, Loss: 2.220579147338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12435 , TFLOPS: 99.64420849844146, Tokens per sec: 81421.8478341983, Loss: 2.2740352153778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12436 , TFLOPS: 98.93609411002723, Tokens per sec: 80843.22933893903, Loss: 2.2754290103912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12437 , TFLOPS: 99.01428814779841, Tokens per sec: 80907.12370009546, Loss: 2.25484299659729 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12438 , TFLOPS: 99.42934619091808, Tokens per sec: 81246.2783116731, Loss: 2.256762981414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12439 , TFLOPS: 99.31775130902173, Tokens per sec: 81155.09126096776, Loss: 2.2699337005615234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12440 , TFLOPS: 99.66313765930303, Tokens per sec: 81437.31533881824, Loss: 2.266995906829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12441 , TFLOPS: 98.85971731483171, Tokens per sec: 80780.81989347115, Loss: 2.243813991546631 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12442 , TFLOPS: 100.2007074787198, Tokens per sec: 81876.57747654193, Loss: 2.2597150802612305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12443 , TFLOPS: 99.07322624473997, Tokens per sec: 80955.283536308, Loss: 2.259751796722412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12444 , TFLOPS: 100.20854324842233, Tokens per sec: 81882.98028567652, Loss: 2.2501978874206543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12445 , TFLOPS: 100.2030587875473, Tokens per sec: 81878.49879151295, Loss: 2.2409536838531494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12446 , TFLOPS: 100.13250513157244, Tokens per sec: 81820.84758300321, Loss: 2.236851215362549 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12447 , TFLOPS: 100.71048943859981, Tokens per sec: 82293.13343891497, Loss: 2.2643485069274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12448 , TFLOPS: 100.53963476103083, Tokens per sec: 82153.52368368268, Loss: 2.244957208633423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12449 , TFLOPS: 100.07476550000851, Tokens per sec: 81773.66704370198, Loss: 2.2683112621307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12450 , TFLOPS: 100.74144899804065, Tokens per sec: 82318.43129190405, Loss: 2.2611353397369385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12451 , TFLOPS: 98.80714723963536, Tokens per sec: 80737.86353175408, Loss: 2.258791446685791 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12452 , TFLOPS: 100.10674544983021, Tokens per sec: 81799.79868393842, Loss: 2.240952491760254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12453 , TFLOPS: 100.59005082086705, Tokens per sec: 82194.7199439997, Loss: 2.264401435852051 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12454 , TFLOPS: 99.39245662854275, Tokens per sec: 81216.1349006344, Loss: 2.227391004562378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12455 , TFLOPS: 100.12463788636107, Tokens per sec: 81814.41905442008, Loss: 2.2743582725524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12456 , TFLOPS: 100.69669104208084, Tokens per sec: 82281.858413917, Loss: 2.2448339462280273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12457 , TFLOPS: 100.06792016866504, Tokens per sec: 81768.07354724672, Loss: 2.262422561645508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12458 , TFLOPS: 100.12328769524568, Tokens per sec: 81813.3157785027, Loss: 2.2608110904693604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12459 , TFLOPS: 99.54658193314617, Tokens per sec: 81342.07465456416, Loss: 2.2405381202697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12460 , TFLOPS: 100.60450779460812, Tokens per sec: 82206.53310939916, Loss: 2.2611231803894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12461 , TFLOPS: 98.96898650429831, Tokens per sec: 80870.1065610235, Loss: 2.2524142265319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12462 , TFLOPS: 99.53740434981582, Tokens per sec: 81334.5754149729, Loss: 2.2603135108947754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12463 , TFLOPS: 100.57103272092361, Tokens per sec: 82179.17976496648, Loss: 2.2448081970214844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12464 , TFLOPS: 99.40297278478572, Tokens per sec: 81224.72792261047, Loss: 2.245448112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12465 , TFLOPS: 100.6840667631164, Tokens per sec: 82271.5427905973, Loss: 2.2483444213867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12466 , TFLOPS: 98.90029962814307, Tokens per sec: 80813.98074635955, Loss: 2.255512237548828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12467 , TFLOPS: 100.65871858550386, Tokens per sec: 82250.83014214996, Loss: 2.2577321529388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12468 , TFLOPS: 98.90421194201036, Tokens per sec: 80817.17759873242, Loss: 2.2640087604522705 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12469 , TFLOPS: 99.81221600445683, Tokens per sec: 81559.1310922614, Loss: 2.264984607696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12470 , TFLOPS: 98.71847356277891, Tokens per sec: 80665.40598773117, Loss: 2.2395074367523193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12471 , TFLOPS: 98.78479209062456, Tokens per sec: 80719.59656402463, Loss: 2.2525458335876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12472 , TFLOPS: 99.68835038219027, Tokens per sec: 81457.91730372261, Loss: 2.2575607299804688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12473 , TFLOPS: 98.20862415647979, Tokens per sec: 80248.79491315281, Loss: 2.253941774368286 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12474 , TFLOPS: 99.5783312513954, Tokens per sec: 81368.01783980576, Loss: 2.2431201934814453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12475 , TFLOPS: 98.40165059312011, Tokens per sec: 80406.52178347415, Loss: 2.244464874267578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12476 , TFLOPS: 100.5981113457928, Tokens per sec: 82201.3064064129, Loss: 2.2497689723968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12477 , TFLOPS: 99.2908356450469, Tokens per sec: 81133.09777906309, Loss: 2.227593421936035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12478 , TFLOPS: 99.40547118423325, Tokens per sec: 81226.76942911357, Loss: 2.2619450092315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12479 , TFLOPS: 98.75943870097879, Tokens per sec: 80698.87965669075, Loss: 2.2570791244506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12480 , TFLOPS: 100.13344071379373, Tokens per sec: 81821.61207132025, Loss: 2.2332870960235596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12481 , TFLOPS: 98.64708103077228, Tokens per sec: 80607.06931202128, Loss: 2.23956561088562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12482 , TFLOPS: 100.08058499161822, Tokens per sec: 81778.42230010331, Loss: 2.25099515914917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12483 , TFLOPS: 98.85343477122507, Tokens per sec: 80775.6862653631, Loss: 2.2442142963409424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12484 , TFLOPS: 100.74413669545395, Tokens per sec: 82320.62747864793, Loss: 2.267078161239624 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12485 , TFLOPS: 100.1162661257051, Tokens per sec: 81807.5782732798, Loss: 2.253085136413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12486 , TFLOPS: 101.34137898772855, Tokens per sec: 82808.64953004985, Loss: 2.269443988800049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12487 , TFLOPS: 100.01586015868708, Tokens per sec: 81725.53397294984, Loss: 2.2385222911834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12488 , TFLOPS: 100.82680382867605, Tokens per sec: 82388.17692124505, Loss: 2.2702746391296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12489 , TFLOPS: 98.12057105516351, Tokens per sec: 80176.84445737881, Loss: 2.2682788372039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12490 , TFLOPS: 100.00039899013191, Tokens per sec: 81712.90025411754, Loss: 2.2541658878326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12491 , TFLOPS: 100.33058196889779, Tokens per sec: 81982.70126573317, Loss: 2.247162342071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12492 , TFLOPS: 100.08481567984369, Tokens per sec: 81781.87930435994, Loss: 2.255552291870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12493 , TFLOPS: 100.19563098561153, Tokens per sec: 81872.4293433426, Loss: 2.2581634521484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12494 , TFLOPS: 100.14648031140302, Tokens per sec: 81832.26706219593, Loss: 2.274050712585449 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12495 , TFLOPS: 100.73562935195163, Tokens per sec: 82313.67590927369, Loss: 2.255403995513916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12496 , TFLOPS: 100.14266686521358, Tokens per sec: 81829.15099714768, Loss: 2.2662062644958496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12497 , TFLOPS: 100.65509960484655, Tokens per sec: 82247.87297989402, Loss: 2.2677385807037354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12498 , TFLOPS: 99.90731304251794, Tokens per sec: 81636.8373300767, Loss: 2.278184175491333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12499 , TFLOPS: 99.42184210509309, Tokens per sec: 81240.14652997315, Loss: 2.2491471767425537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12500 , TFLOPS: 99.95781017804764, Tokens per sec: 81678.09983942992, Loss: 2.2557191848754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12501 , TFLOPS: 100.48619651790723, Tokens per sec: 82109.85791960364, Loss: 2.254387378692627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12502 , TFLOPS: 98.91336281756284, Tokens per sec: 80824.65501471084, Loss: 2.273080348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12503 , TFLOPS: 100.78069689002159, Tokens per sec: 82350.5017547723, Loss: 2.2473881244659424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12504 , TFLOPS: 98.97618503247878, Tokens per sec: 80875.98866370635, Loss: 2.2728703022003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12505 , TFLOPS: 100.5693717223886, Tokens per sec: 82177.8225203056, Loss: 2.2539546489715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12506 , TFLOPS: 98.13746406546855, Tokens per sec: 80190.64817096383, Loss: 2.2609286308288574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12507 , TFLOPS: 100.44594644553959, Tokens per sec: 82076.968549343, Loss: 2.2543625831604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12508 , TFLOPS: 97.77994218482762, Tokens per sec: 79898.50783886017, Loss: 2.2872185707092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12509 , TFLOPS: 99.47034613873063, Tokens per sec: 81279.78042446329, Loss: 2.2647740840911865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12510 , TFLOPS: 98.8575455685101, Tokens per sec: 80779.04530364604, Loss: 2.257838487625122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12511 , TFLOPS: 98.98914082009988, Tokens per sec: 80886.57517128318, Loss: 2.2468905448913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12512 , TFLOPS: 99.00006028905864, Tokens per sec: 80895.49775046152, Loss: 2.2677688598632812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12513 , TFLOPS: 98.83071382349394, Tokens per sec: 80757.12039408275, Loss: 2.249724864959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12514 , TFLOPS: 100.17144954298332, Tokens per sec: 81852.67006408538, Loss: 2.2379276752471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12515 , TFLOPS: 99.49837811562122, Tokens per sec: 81302.68607438786, Loss: 2.2495462894439697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12516 , TFLOPS: 99.64704881089692, Tokens per sec: 81424.1687266218, Loss: 2.2777559757232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12517 , TFLOPS: 99.48870934564316, Tokens per sec: 81294.7854735425, Loss: 2.2562928199768066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12518 , TFLOPS: 100.77034415259789, Tokens per sec: 82342.0422665202, Loss: 2.240098237991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12519 , TFLOPS: 98.93976439298348, Tokens per sec: 80846.22842162407, Loss: 2.2643089294433594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12520 , TFLOPS: 100.27812385074587, Tokens per sec: 81939.83638699913, Loss: 2.2747678756713867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12521 , TFLOPS: 98.93137308704011, Tokens per sec: 80839.3716695263, Loss: 2.2452504634857178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12522 , TFLOPS: 100.61651482239934, Tokens per sec: 82216.34436089575, Loss: 2.2562010288238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12523 , TFLOPS: 100.0144324655059, Tokens per sec: 81724.36736809943, Loss: 2.266676902770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12524 , TFLOPS: 100.48453447810013, Tokens per sec: 82108.49982409255, Loss: 2.2580487728118896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12525 , TFLOPS: 100.04480405962991, Tokens per sec: 81749.18477949253, Loss: 2.243910074234009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12526 , TFLOPS: 100.65520233743251, Tokens per sec: 82247.95692533458, Loss: 2.241196393966675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12527 , TFLOPS: 98.38008642515254, Tokens per sec: 80388.9011467168, Loss: 2.235691547393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12528 , TFLOPS: 100.22266954032513, Tokens per sec: 81894.5232424334, Loss: 2.2427079677581787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12529 , TFLOPS: 100.15797652973528, Tokens per sec: 81841.66091813419, Loss: 2.23982834815979 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12530 , TFLOPS: 100.10462028708541, Tokens per sec: 81798.06215875309, Loss: 2.263498067855835 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12531 , TFLOPS: 98.89052301725997, Tokens per sec: 80805.9920259346, Loss: 2.2581517696380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12532 , TFLOPS: 100.745882153592, Tokens per sec: 82322.05373742463, Loss: 2.2750020027160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12533 , TFLOPS: 101.31074681689074, Tokens per sec: 82783.61919471607, Loss: 2.2451744079589844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12534 , TFLOPS: 100.04774468881412, Tokens per sec: 81751.58764329748, Loss: 2.25425386428833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12535 , TFLOPS: 99.48134089621108, Tokens per sec: 81288.76452383079, Loss: 2.264600992202759 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12536 , TFLOPS: 100.62840711068718, Tokens per sec: 82226.06185579045, Loss: 2.26533579826355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12537 , TFLOPS: 100.06569578216774, Tokens per sec: 81766.25594377896, Loss: 2.2511823177337646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12538 , TFLOPS: 99.44737308229395, Tokens per sec: 81261.00854866978, Loss: 2.243340015411377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12539 , TFLOPS: 100.51462840139797, Tokens per sec: 82133.09034350573, Loss: 2.278256893157959 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12540 , TFLOPS: 98.03957467531353, Tokens per sec: 80110.6602303711, Loss: 2.2660059928894043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12541 , TFLOPS: 101.18315217134902, Tokens per sec: 82679.35832526554, Loss: 2.2542636394500732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12542 , TFLOPS: 98.22555085555797, Tokens per sec: 80262.62615470162, Loss: 2.261568069458008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12543 , TFLOPS: 100.72934451936769, Tokens per sec: 82308.54041078336, Loss: 2.2436182498931885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12544 , TFLOPS: 98.21899524260344, Tokens per sec: 80257.26939459996, Loss: 2.2549524307250977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12545 , TFLOPS: 99.9098177481558, Tokens per sec: 81638.88398953025, Loss: 2.2610647678375244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12546 , TFLOPS: 99.5080039559207, Tokens per sec: 81310.55159628774, Loss: 2.2594802379608154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12547 , TFLOPS: 98.77470801187255, Tokens per sec: 80711.35660368908, Loss: 2.250690221786499 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12548 , TFLOPS: 99.00288170046844, Tokens per sec: 80897.80319835407, Loss: 2.2451891899108887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12549 , TFLOPS: 98.38485431658256, Tokens per sec: 80392.79711354071, Loss: 2.258178234100342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12550 , TFLOPS: 99.93145921790347, Tokens per sec: 81656.56779156206, Loss: 2.2570908069610596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12551 , TFLOPS: 97.60690650093991, Tokens per sec: 79757.11592722074, Loss: 2.249321460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12552 , TFLOPS: 100.57095596875753, Tokens per sec: 82179.11704879579, Loss: 2.2593350410461426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12553 , TFLOPS: 98.56936002801989, Tokens per sec: 80543.5614799557, Loss: 2.2542343139648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12554 , TFLOPS: 100.54911177330519, Tokens per sec: 82161.26759437207, Loss: 2.2467992305755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12555 , TFLOPS: 98.86269794237187, Tokens per sec: 80783.25544096238, Loss: 2.2593798637390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12556 , TFLOPS: 100.6861479998505, Tokens per sec: 82273.24342270853, Loss: 2.24621844291687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12557 , TFLOPS: 98.78860513875294, Tokens per sec: 80722.71230380691, Loss: 2.252833604812622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12558 , TFLOPS: 100.2015361237586, Tokens per sec: 81877.25458373434, Loss: 2.2440624237060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12559 , TFLOPS: 98.23678780363497, Tokens per sec: 80271.8081542401, Loss: 2.2587242126464844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12560 , TFLOPS: 100.73098465631864, Tokens per sec: 82309.88060890688, Loss: 2.253661870956421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12561 , TFLOPS: 100.133435393333, Tokens per sec: 81821.60772383484, Loss: 2.2610971927642822 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12562 , TFLOPS: 100.7258828777061, Tokens per sec: 82305.7118142711, Loss: 2.2466249465942383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12563 , TFLOPS: 100.04656166882597, Tokens per sec: 81750.62096721152, Loss: 2.2606608867645264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12564 , TFLOPS: 100.1633684343769, Tokens per sec: 81846.06678221683, Loss: 2.2400054931640625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12565 , TFLOPS: 99.94033044458394, Tokens per sec: 81663.81669924836, Loss: 2.240295886993408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12566 , TFLOPS: 96.85000605553317, Tokens per sec: 79138.63308892807, Loss: 2.2553515434265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12567 , TFLOPS: 100.07946950710469, Tokens per sec: 81777.51080899219, Loss: 2.2493293285369873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12568 , TFLOPS: 100.54821049370433, Tokens per sec: 82160.53113560921, Loss: 2.228625535964966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12569 , TFLOPS: 100.13414333154375, Tokens per sec: 81822.1861983708, Loss: 2.247048854827881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12570 , TFLOPS: 100.65503074431119, Tokens per sec: 82247.81671217796, Loss: 2.2688136100769043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12571 , TFLOPS: 101.33681517249198, Tokens per sec: 82804.920319137, Loss: 2.2584550380706787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12572 , TFLOPS: 98.61954222820309, Tokens per sec: 80584.56664753014, Loss: 2.2626941204071045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12573 , TFLOPS: 100.76576372426649, Tokens per sec: 82338.29948061991, Loss: 2.2662429809570312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12574 , TFLOPS: 99.95678584096173, Tokens per sec: 81677.26282722825, Loss: 2.270470142364502 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12575 , TFLOPS: 100.07795467030118, Tokens per sec: 81776.27299684465, Loss: 2.249955892562866 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12576 , TFLOPS: 98.87448104739248, Tokens per sec: 80792.88371939884, Loss: 2.2531323432922363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12577 , TFLOPS: 100.59421518961663, Tokens per sec: 82198.12275690537, Loss: 2.2544467449188232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12578 , TFLOPS: 98.75243568401332, Tokens per sec: 80693.15731125457, Loss: 2.2181613445281982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12579 , TFLOPS: 101.22768014826724, Tokens per sec: 82715.74328145751, Loss: 2.2745168209075928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12580 , TFLOPS: 98.46080788432123, Tokens per sec: 80454.86072895875, Loss: 2.254194736480713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12581 , TFLOPS: 99.5401187800003, Tokens per sec: 81336.7934457523, Loss: 2.2500674724578857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12582 , TFLOPS: 99.40198591065645, Tokens per sec: 81223.92152235506, Loss: 2.2659494876861572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12583 , TFLOPS: 99.43210990843903, Tokens per sec: 81248.53661640389, Loss: 2.276233434677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12584 , TFLOPS: 99.4516913682749, Tokens per sec: 81264.53713130738, Loss: 2.2572007179260254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12585 , TFLOPS: 98.86528384865092, Tokens per sec: 80785.36845155015, Loss: 2.2663815021514893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12586 , TFLOPS: 100.1798403926717, Tokens per sec: 81859.52644336542, Loss: 2.2402126789093018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12587 , TFLOPS: 97.75956310547083, Tokens per sec: 79881.85556851355, Loss: 2.257106304168701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12588 , TFLOPS: 100.7637691541388, Tokens per sec: 82336.66966602376, Loss: 2.25136137008667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12589 , TFLOPS: 97.71203709409329, Tokens per sec: 79843.0208411886, Loss: 2.2386345863342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12590 , TFLOPS: 101.32623876327479, Tokens per sec: 82796.27806290465, Loss: 2.2307844161987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12591 , TFLOPS: 98.34649399723587, Tokens per sec: 80361.45190912006, Loss: 2.2366790771484375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12592 , TFLOPS: 100.62913824177124, Tokens per sec: 82226.65928182025, Loss: 2.2484214305877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12593 , TFLOPS: 99.35880555959392, Tokens per sec: 81188.63774594072, Loss: 2.2396817207336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12594 , TFLOPS: 100.2839949034464, Tokens per sec: 81944.63377529508, Loss: 2.2745299339294434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12595 , TFLOPS: 99.27702726894454, Tokens per sec: 81121.81459949064, Loss: 2.2514781951904297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12596 , TFLOPS: 99.21733323240643, Tokens per sec: 81073.03706557414, Loss: 2.2482166290283203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12597 , TFLOPS: 98.4010207864035, Tokens per sec: 80406.00715219336, Loss: 2.263888120651245 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12598 , TFLOPS: 100.07663575340811, Tokens per sec: 81775.19527589939, Loss: 2.233798027038574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12599 , TFLOPS: 100.64223361905425, Tokens per sec: 82237.3598517032, Loss: 2.2585461139678955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12600 , TFLOPS: 101.33814337028352, Tokens per sec: 82806.00562374332, Loss: 2.2509703636169434 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/optimizer.pt +[rank0]:[2024-08-29 11:55:13,992] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007701050999457948, 'preprocessing_with_comm': 0.0015968380030244589, 'state_converting': 2.39902940599859, : 2.409944506995089}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012600, took 14.72s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010600 + eval ppl=7.476416110992432, eval loss=2.011753559112549 +------------------------------------------------------------------ +iteration: 12601 , TFLOPS: 98.00065963122212, Tokens per sec: 80078.86174608165, Loss: 2.2635324001312256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12602 , TFLOPS: 99.02217471240897, Tokens per sec: 80913.56801505698, Loss: 2.237201690673828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12603 , TFLOPS: 99.92664749586248, Tokens per sec: 81652.63600961762, Loss: 2.28017520904541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12604 , TFLOPS: 98.86263774579062, Tokens per sec: 80783.20625278626, Loss: 2.259913444519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12605 , TFLOPS: 99.44257703882737, Tokens per sec: 81257.0895780921, Loss: 2.253995418548584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12606 , TFLOPS: 99.51425425348768, Tokens per sec: 81315.65887532668, Loss: 2.2527248859405518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12607 , TFLOPS: 99.98154166430169, Tokens per sec: 81697.4914477509, Loss: 2.251361608505249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12608 , TFLOPS: 98.9086543946663, Tokens per sec: 80820.8076411565, Loss: 2.251222610473633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12609 , TFLOPS: 101.33451099348767, Tokens per sec: 82803.03751515773, Loss: 2.264732837677002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12610 , TFLOPS: 100.06125402784359, Tokens per sec: 81762.62647197978, Loss: 2.2574822902679443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12611 , TFLOPS: 101.28136573948946, Tokens per sec: 82759.61116003526, Loss: 2.2709832191467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12612 , TFLOPS: 99.47453269306583, Tokens per sec: 81283.20136578208, Loss: 2.255443572998047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12613 , TFLOPS: 101.02839827480302, Tokens per sec: 82552.90493267811, Loss: 2.2659671306610107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12614 , TFLOPS: 99.91261081474637, Tokens per sec: 81641.16627614133, Loss: 2.2738289833068848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12615 , TFLOPS: 100.90164946979618, Tokens per sec: 82449.3352213025, Loss: 2.2439017295837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12616 , TFLOPS: 100.31785537177784, Tokens per sec: 81972.30203561469, Loss: 2.2588324546813965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12617 , TFLOPS: 99.99245243740708, Tokens per sec: 81706.4069213236, Loss: 2.2452614307403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12618 , TFLOPS: 100.63771922128016, Tokens per sec: 82233.67102107104, Loss: 2.256124973297119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12619 , TFLOPS: 101.27350213058357, Tokens per sec: 82753.18560277094, Loss: 2.2498960494995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12620 , TFLOPS: 100.04880480499979, Tokens per sec: 81752.45389152261, Loss: 2.2553622722625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12621 , TFLOPS: 100.42195919839753, Tokens per sec: 82057.36795221655, Loss: 2.2259953022003174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12622 , TFLOPS: 100.06483174072959, Tokens per sec: 81765.54991327746, Loss: 2.259949207305908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12623 , TFLOPS: 101.2445412243212, Tokens per sec: 82729.52090074391, Loss: 2.2486796379089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12624 , TFLOPS: 99.64037640677729, Tokens per sec: 81418.7165334527, Loss: 2.2405107021331787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12625 , TFLOPS: 100.57321828446001, Tokens per sec: 82180.96564519347, Loss: 2.2683277130126953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12626 , TFLOPS: 100.6741803322799, Tokens per sec: 82263.46433346142, Loss: 2.2657599449157715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12627 , TFLOPS: 99.31879804493542, Tokens per sec: 81155.94657582822, Loss: 2.2726986408233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12628 , TFLOPS: 100.65569577974284, Tokens per sec: 82248.36012974867, Loss: 2.243450880050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12629 , TFLOPS: 99.92076197095896, Tokens per sec: 81647.82679571206, Loss: 2.2360804080963135 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12630 , TFLOPS: 100.66073393862227, Tokens per sec: 82252.47693906282, Loss: 2.244349718093872 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12631 , TFLOPS: 98.73665983035612, Tokens per sec: 80680.26645512467, Loss: 2.2501068115234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12632 , TFLOPS: 100.65351921182479, Tokens per sec: 82246.581600073, Loss: 2.2602107524871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12633 , TFLOPS: 98.73679077941883, Tokens per sec: 80680.37345697475, Loss: 2.2477035522460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12634 , TFLOPS: 101.26131797497693, Tokens per sec: 82743.2296155767, Loss: 2.2774014472961426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12635 , TFLOPS: 98.22263571769815, Tokens per sec: 80260.24412051402, Loss: 2.2639644145965576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12636 , TFLOPS: 100.65002031165605, Tokens per sec: 82243.72255867542, Loss: 2.2505180835723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12637 , TFLOPS: 99.21280342094609, Tokens per sec: 81069.33564002218, Loss: 2.2469334602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12638 , TFLOPS: 100.07146252670263, Tokens per sec: 81770.96809918764, Loss: 2.2648587226867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12639 , TFLOPS: 98.86871159410727, Tokens per sec: 80788.16935060054, Loss: 2.2566895484924316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12640 , TFLOPS: 99.55515643097247, Tokens per sec: 81349.08109746523, Loss: 2.2592685222625732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12641 , TFLOPS: 100.0808351102497, Tokens per sec: 81778.62667847572, Loss: 2.2407989501953125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12642 , TFLOPS: 98.75178332833467, Tokens per sec: 80692.62425463642, Loss: 2.2536087036132812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12643 , TFLOPS: 100.8069167227686, Tokens per sec: 82371.92665506843, Loss: 2.277137279510498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12644 , TFLOPS: 99.61650286334438, Tokens per sec: 81399.20884655399, Loss: 2.2469241619110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12645 , TFLOPS: 100.77097541563764, Tokens per sec: 82342.55808780016, Loss: 2.2655463218688965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12646 , TFLOPS: 99.50426915120651, Tokens per sec: 81307.49979121335, Loss: 2.244182586669922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12647 , TFLOPS: 101.27886591886639, Tokens per sec: 82757.56849225303, Loss: 2.2361464500427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12648 , TFLOPS: 98.69490002291778, Tokens per sec: 80646.14344147373, Loss: 2.275142192840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12649 , TFLOPS: 101.29408616918994, Tokens per sec: 82770.0053505965, Loss: 2.264718532562256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12650 , TFLOPS: 99.57824539718865, Tokens per sec: 81367.94768612333, Loss: 2.2647552490234375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12651 , TFLOPS: 101.28720964483706, Tokens per sec: 82764.38636553028, Loss: 2.258571147918701 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12652 , TFLOPS: 98.80395891723839, Tokens per sec: 80735.2582714488, Loss: 2.2498741149902344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12653 , TFLOPS: 100.2649511756519, Tokens per sec: 81929.07265508502, Loss: 2.2591779232025146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12654 , TFLOPS: 100.67681907221133, Tokens per sec: 82265.6205157866, Loss: 2.2677202224731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12655 , TFLOPS: 99.94186667319845, Tokens per sec: 81665.07199119532, Loss: 2.2631852626800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12656 , TFLOPS: 100.07268526130224, Tokens per sec: 81771.96722710498, Loss: 2.247087001800537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12657 , TFLOPS: 101.25151283197425, Tokens per sec: 82735.21758082237, Loss: 2.243203639984131 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12658 , TFLOPS: 99.42757742725557, Tokens per sec: 81244.83300935243, Loss: 2.2400646209716797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12659 , TFLOPS: 101.34720097018109, Tokens per sec: 82813.40682178299, Loss: 2.2673959732055664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12660 , TFLOPS: 100.00851163538616, Tokens per sec: 81719.52930539285, Loss: 2.2562642097473145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12661 , TFLOPS: 99.94659119304947, Tokens per sec: 81668.93251798555, Loss: 2.2678112983703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12662 , TFLOPS: 100.17104836008873, Tokens per sec: 81852.34224721484, Loss: 2.2544026374816895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12663 , TFLOPS: 100.0510300304375, Tokens per sec: 81754.27218051016, Loss: 2.2569031715393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12664 , TFLOPS: 100.80179183401673, Tokens per sec: 82367.73897654288, Loss: 2.253917694091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12665 , TFLOPS: 99.45404499888168, Tokens per sec: 81266.46034346404, Loss: 2.2558670043945312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12666 , TFLOPS: 100.05583584428858, Tokens per sec: 81758.19913472053, Loss: 2.2536492347717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12667 , TFLOPS: 98.75118293405097, Tokens per sec: 80692.1336570117, Loss: 2.2548117637634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12668 , TFLOPS: 100.63636209518043, Tokens per sec: 82232.56207839944, Loss: 2.2561686038970947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12669 , TFLOPS: 98.19746127303878, Tokens per sec: 80239.67343373499, Loss: 2.2542269229888916 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12670 , TFLOPS: 99.94157315549292, Tokens per sec: 81664.83215032231, Loss: 2.2416646480560303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12671 , TFLOPS: 98.89645316701001, Tokens per sec: 80810.837703951, Loss: 2.248404026031494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12672 , TFLOPS: 100.8005039015414, Tokens per sec: 82366.68657376297, Loss: 2.2442400455474854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12673 , TFLOPS: 98.25639057653635, Tokens per sec: 80287.82608459808, Loss: 2.251241445541382 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12674 , TFLOPS: 100.55690076375456, Tokens per sec: 82167.63217897469, Loss: 2.261037588119507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12675 , TFLOPS: 99.49274976762165, Tokens per sec: 81298.08700635085, Loss: 2.2565829753875732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12676 , TFLOPS: 99.32340807490361, Tokens per sec: 81159.71354998797, Loss: 2.265744924545288 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12677 , TFLOPS: 99.53267976718925, Tokens per sec: 81330.71483688717, Loss: 2.255598545074463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12678 , TFLOPS: 99.08761540159503, Tokens per sec: 80967.04128678412, Loss: 2.2412915229797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12679 , TFLOPS: 100.81230776351222, Tokens per sec: 82376.33181323776, Loss: 2.2730908393859863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12680 , TFLOPS: 98.31839403500892, Tokens per sec: 80338.49070662704, Loss: 2.2658538818359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12681 , TFLOPS: 99.90519207441346, Tokens per sec: 81635.10423243996, Loss: 2.2427828311920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12682 , TFLOPS: 100.02265268400411, Tokens per sec: 81731.08432024156, Loss: 2.2438766956329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12683 , TFLOPS: 100.18826577513711, Tokens per sec: 81866.41104026657, Loss: 2.260780096054077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12684 , TFLOPS: 99.45767075974366, Tokens per sec: 81269.42304599973, Loss: 2.2673521041870117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12685 , TFLOPS: 101.36672288286184, Tokens per sec: 82829.35867917318, Loss: 2.241652011871338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12686 , TFLOPS: 98.8306053584317, Tokens per sec: 80757.03176448827, Loss: 2.2620840072631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12687 , TFLOPS: 101.29627914121521, Tokens per sec: 82771.79728449049, Loss: 2.2784581184387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12688 , TFLOPS: 99.34933313649924, Tokens per sec: 81180.8975851882, Loss: 2.250859260559082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12689 , TFLOPS: 101.19457523924636, Tokens per sec: 82688.69240810038, Loss: 2.2567758560180664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12690 , TFLOPS: 99.49389435659107, Tokens per sec: 81299.0222794621, Loss: 2.2571892738342285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12691 , TFLOPS: 100.74979877255635, Tokens per sec: 82325.25410760313, Loss: 2.238588571548462 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12692 , TFLOPS: 99.99758266613024, Tokens per sec: 81710.59896327714, Loss: 2.257831335067749 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12693 , TFLOPS: 100.56515250781977, Tokens per sec: 82174.37489146918, Loss: 2.2699809074401855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12694 , TFLOPS: 100.088436012714, Tokens per sec: 81784.83757154393, Loss: 2.2538726329803467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12695 , TFLOPS: 101.33393971844745, Tokens per sec: 82802.57071161644, Loss: 2.239884853363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12696 , TFLOPS: 99.44323187699732, Tokens per sec: 81257.6246632178, Loss: 2.2577450275421143 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12697 , TFLOPS: 99.84769697930973, Tokens per sec: 81588.12351017527, Loss: 2.2393412590026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12698 , TFLOPS: 100.82346155618515, Tokens per sec: 82385.44586435497, Loss: 2.2445716857910156 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12699 , TFLOPS: 100.61012244492608, Tokens per sec: 82211.1209847079, Loss: 2.2608890533447266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12700 , TFLOPS: 99.42600832963637, Tokens per sec: 81243.55085929562, Loss: 2.2434592247009277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12701 , TFLOPS: 100.01222310855624, Tokens per sec: 81722.56204566195, Loss: 2.275486707687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12702 , TFLOPS: 100.72700169729063, Tokens per sec: 82306.6260305546, Loss: 2.27109956741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12703 , TFLOPS: 98.73097542238007, Tokens per sec: 80675.62157903776, Loss: 2.2661893367767334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12704 , TFLOPS: 100.72937919954735, Tokens per sec: 82308.56874885093, Loss: 2.2679877281188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12705 , TFLOPS: 99.00184406693961, Tokens per sec: 80896.9553212866, Loss: 2.263021230697632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12706 , TFLOPS: 100.65633886478086, Tokens per sec: 82248.88561108774, Loss: 2.2497830390930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12707 , TFLOPS: 98.29224451140522, Tokens per sec: 80317.12325774183, Loss: 2.258384943008423 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12708 , TFLOPS: 99.38617927756431, Tokens per sec: 81211.00551555652, Loss: 2.2482144832611084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12709 , TFLOPS: 99.5260960651797, Tokens per sec: 81325.3351244956, Loss: 2.229600191116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12710 , TFLOPS: 100.11161167222755, Tokens per sec: 81803.77499952701, Loss: 2.2446255683898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12711 , TFLOPS: 98.28900943479219, Tokens per sec: 80314.47979336305, Loss: 2.267022132873535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12712 , TFLOPS: 100.65123000017337, Tokens per sec: 82244.71102630308, Loss: 2.249220848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12713 , TFLOPS: 98.93542350604129, Tokens per sec: 80842.68137115923, Loss: 2.274449110031128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12714 , TFLOPS: 100.86202296239203, Tokens per sec: 82416.95538202554, Loss: 2.267500400543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12715 , TFLOPS: 98.44511577842609, Tokens per sec: 80442.03830528108, Loss: 2.2352511882781982 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12716 , TFLOPS: 98.90089586688465, Tokens per sec: 80814.46794838384, Loss: 2.2597765922546387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12717 , TFLOPS: 99.85564554803757, Tokens per sec: 81594.61849029716, Loss: 2.253168821334839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12718 , TFLOPS: 99.0097526982696, Tokens per sec: 80903.4176675326, Loss: 2.2331583499908447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12719 , TFLOPS: 101.31322239770067, Tokens per sec: 82785.64205552297, Loss: 2.2500522136688232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12720 , TFLOPS: 99.45314825604414, Tokens per sec: 81265.72759180727, Loss: 2.2742819786071777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12721 , TFLOPS: 101.31826435006252, Tokens per sec: 82789.76196458924, Loss: 2.258962631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12722 , TFLOPS: 99.34390702581908, Tokens per sec: 81176.46377047093, Loss: 2.2565431594848633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12723 , TFLOPS: 101.34455333312923, Tokens per sec: 82811.24336939168, Loss: 2.253798007965088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12724 , TFLOPS: 98.66396819986241, Tokens per sec: 80620.86825259922, Loss: 2.2479398250579834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12725 , TFLOPS: 101.26334515371805, Tokens per sec: 82744.8860755103, Loss: 2.265118360519409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12726 , TFLOPS: 98.90632909013081, Tokens per sec: 80818.9075749619, Loss: 2.2673277854919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12727 , TFLOPS: 101.24443727248264, Tokens per sec: 82729.43595902067, Loss: 2.2655751705169678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12728 , TFLOPS: 98.92237536938504, Tokens per sec: 80832.01940280848, Loss: 2.241563081741333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12729 , TFLOPS: 98.83665168971444, Tokens per sec: 80761.97237742582, Loss: 2.2338671684265137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12730 , TFLOPS: 101.34493217259205, Tokens per sec: 82811.55292886894, Loss: 2.247800588607788 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12731 , TFLOPS: 100.18262214238712, Tokens per sec: 81861.79948266651, Loss: 2.2517881393432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12732 , TFLOPS: 99.42594666318804, Tokens per sec: 81243.50047005324, Loss: 2.268264055252075 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12733 , TFLOPS: 100.74117106100749, Tokens per sec: 82318.20418239952, Loss: 2.2369422912597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12734 , TFLOPS: 100.63842935634072, Tokens per sec: 82234.25129070954, Loss: 2.2624402046203613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12735 , TFLOPS: 99.32640842788116, Tokens per sec: 81162.16521564187, Loss: 2.2405600547790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12736 , TFLOPS: 100.70078812863876, Tokens per sec: 82285.20624881187, Loss: 2.2708792686462402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12737 , TFLOPS: 100.10243562059968, Tokens per sec: 81796.27701152928, Loss: 2.261073112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12738 , TFLOPS: 99.25805340319786, Tokens per sec: 81106.31056535828, Loss: 2.276238441467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12739 , TFLOPS: 99.50776699110665, Tokens per sec: 81310.35796623815, Loss: 2.252215623855591 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12740 , TFLOPS: 99.61650464328513, Tokens per sec: 81399.21030098939, Loss: 2.2483065128326416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12741 , TFLOPS: 100.70202783802155, Tokens per sec: 82286.2192472615, Loss: 2.254490613937378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12742 , TFLOPS: 99.44419519895366, Tokens per sec: 81258.41181838645, Loss: 2.2566206455230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12743 , TFLOPS: 100.03117835476792, Tokens per sec: 81738.05086529296, Loss: 2.259082794189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12744 , TFLOPS: 99.94645305853054, Tokens per sec: 81668.81964471423, Loss: 2.263958215713501 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12745 , TFLOPS: 97.90734746526495, Tokens per sec: 80002.6139732092, Loss: 2.2449593544006348 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12746 , TFLOPS: 100.67874598631279, Tokens per sec: 82267.1950469021, Loss: 2.2565670013427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12747 , TFLOPS: 99.58032601730369, Tokens per sec: 81369.64781437926, Loss: 2.2538955211639404 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12748 , TFLOPS: 100.22605410591873, Tokens per sec: 81897.28885810639, Loss: 2.255747079849243 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12749 , TFLOPS: 98.65160876508311, Tokens per sec: 80610.76904028091, Loss: 2.2768173217773438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12750 , TFLOPS: 100.72602720653194, Tokens per sec: 82305.82974907004, Loss: 2.2503395080566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12751 , TFLOPS: 100.05801774769334, Tokens per sec: 81759.98202415975, Loss: 2.244626522064209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12752 , TFLOPS: 101.28328769591221, Tokens per sec: 82761.18164010384, Loss: 2.244555950164795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12753 , TFLOPS: 98.53868792227404, Tokens per sec: 80518.49851278051, Loss: 2.2583422660827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12754 , TFLOPS: 100.82404241771827, Tokens per sec: 82385.9205012664, Loss: 2.2570157051086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12755 , TFLOPS: 100.55026422356221, Tokens per sec: 82162.2092911438, Loss: 2.2507994174957275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12756 , TFLOPS: 97.72219000059283, Tokens per sec: 79851.31704244844, Loss: 2.244323253631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12757 , TFLOPS: 100.5261374648882, Tokens per sec: 82142.49469555324, Loss: 2.2592477798461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12758 , TFLOPS: 100.11214899293473, Tokens per sec: 81804.21405810872, Loss: 2.2673990726470947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12759 , TFLOPS: 100.58270225949326, Tokens per sec: 82188.71524533238, Loss: 2.2668306827545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12760 , TFLOPS: 100.14458740276987, Tokens per sec: 81830.72031782396, Loss: 2.2588181495666504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12761 , TFLOPS: 100.68848251301391, Tokens per sec: 82275.15101351007, Loss: 2.2424159049987793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12762 , TFLOPS: 99.35128350041988, Tokens per sec: 81182.4912777546, Loss: 2.2430667877197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12763 , TFLOPS: 101.28930642842668, Tokens per sec: 82766.09970137739, Loss: 2.2519659996032715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12764 , TFLOPS: 98.47346853310148, Tokens per sec: 80465.20607099117, Loss: 2.2685654163360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12765 , TFLOPS: 101.26935588267675, Tokens per sec: 82749.79759687238, Loss: 2.239739418029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12766 , TFLOPS: 98.82751323158031, Tokens per sec: 80754.50510803956, Loss: 2.2458345890045166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12767 , TFLOPS: 98.99194122454774, Tokens per sec: 80888.86345384635, Loss: 2.257065534591675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12768 , TFLOPS: 101.35164913741886, Tokens per sec: 82817.041533739, Loss: 2.262687921524048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12769 , TFLOPS: 99.45834410660169, Tokens per sec: 81269.97325505094, Loss: 2.258643627166748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12770 , TFLOPS: 99.49988683987823, Tokens per sec: 81303.91889181628, Loss: 2.257750988006592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12771 , TFLOPS: 100.1427120229346, Tokens per sec: 81829.18789668399, Loss: 2.2534127235412598 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12772 , TFLOPS: 100.70738048572697, Tokens per sec: 82290.59303349101, Loss: 2.2423226833343506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12773 , TFLOPS: 99.411580909221, Tokens per sec: 81231.76184267938, Loss: 2.264918088912964 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12774 , TFLOPS: 100.7163220753849, Tokens per sec: 82297.8994365775, Loss: 2.2662353515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12775 , TFLOPS: 100.17715990648584, Tokens per sec: 81857.33614910112, Loss: 2.260061502456665 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12776 , TFLOPS: 100.11811736055392, Tokens per sec: 81809.0909649298, Loss: 2.261584758758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12777 , TFLOPS: 99.48753673527243, Tokens per sec: 81293.8273034229, Loss: 2.2481274604797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12778 , TFLOPS: 99.21678634555325, Tokens per sec: 81072.59019024829, Loss: 2.256779193878174 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12779 , TFLOPS: 100.77350772558853, Tokens per sec: 82344.62730344848, Loss: 2.2715225219726562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12780 , TFLOPS: 98.79664262607882, Tokens per sec: 80729.27994160423, Loss: 2.2545628547668457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12781 , TFLOPS: 100.22158185210242, Tokens per sec: 81893.63446438703, Loss: 2.2851529121398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12782 , TFLOPS: 100.75154650688404, Tokens per sec: 82326.68222631299, Loss: 2.2462987899780273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12783 , TFLOPS: 97.77508230694863, Tokens per sec: 79894.53670754087, Loss: 2.237617015838623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12784 , TFLOPS: 100.71783165351945, Tokens per sec: 82299.13295173124, Loss: 2.248046636581421 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12785 , TFLOPS: 99.55560608482367, Tokens per sec: 81349.44852120215, Loss: 2.2643208503723145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12786 , TFLOPS: 99.38174627883515, Tokens per sec: 81207.38319817939, Loss: 2.2485640048980713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12787 , TFLOPS: 99.42590828791418, Tokens per sec: 81243.46911262911, Loss: 2.2510857582092285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12788 , TFLOPS: 100.63713987384992, Tokens per sec: 82233.19762137208, Loss: 2.2493700981140137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12789 , TFLOPS: 100.22856607237892, Tokens per sec: 81899.34145056478, Loss: 2.2364141941070557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12790 , TFLOPS: 101.27633696419927, Tokens per sec: 82755.50201829345, Loss: 2.2399117946624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12791 , TFLOPS: 98.90918316930002, Tokens per sec: 80821.23971652158, Loss: 2.2561984062194824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12792 , TFLOPS: 100.72245268695663, Tokens per sec: 82302.90891710874, Loss: 2.2707173824310303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12793 , TFLOPS: 100.8092220727656, Tokens per sec: 82373.81041589596, Loss: 2.248464345932007 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12794 , TFLOPS: 97.49336429732999, Tokens per sec: 79664.33766981473, Loss: 2.239117383956909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12795 , TFLOPS: 100.54493036065958, Tokens per sec: 82157.8508544602, Loss: 2.237847089767456 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12796 , TFLOPS: 100.0910326242318, Tokens per sec: 81786.95932965785, Loss: 2.2481327056884766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12797 , TFLOPS: 101.27513700124284, Tokens per sec: 82754.52149767196, Loss: 2.2516326904296875 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12798 , TFLOPS: 99.09298405489659, Tokens per sec: 80971.42815159824, Loss: 2.2556312084198 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12799 , TFLOPS: 100.6541063890401, Tokens per sec: 82247.06139769091, Loss: 2.237332582473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12800 , TFLOPS: 99.27004246875858, Tokens per sec: 81116.10713945393, Loss: 2.258528232574463 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/optimizer.pt +[rank0]:[2024-08-29 12:49:08,196] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007863716993597336, 'preprocessing_with_comm': 0.0016701770000508986, 'state_converting': 2.3529852170031518, : 2.3642247739990125}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0012800, took 14.41s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0010800 + eval ppl=7.507938861846924, eval loss=2.015960931777954 +------------------------------------------------------------------ +iteration: 12801 , TFLOPS: 98.63824030121083, Tokens per sec: 80599.84532431599, Loss: 2.250072717666626 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12802 , TFLOPS: 99.04688146764376, Tokens per sec: 80933.75654076778, Loss: 2.255183696746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12803 , TFLOPS: 99.54730047103848, Tokens per sec: 81342.66179037277, Loss: 2.2664456367492676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12804 , TFLOPS: 99.06815240671668, Tokens per sec: 80951.13757264696, Loss: 2.2531206607818604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12805 , TFLOPS: 100.13296937665419, Tokens per sec: 81821.22692961023, Loss: 2.263882637023926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12806 , TFLOPS: 100.02619903507578, Tokens per sec: 81733.9821349934, Loss: 2.260462760925293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12807 , TFLOPS: 100.68873077319951, Tokens per sec: 82275.3538732985, Loss: 2.2760374546051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12808 , TFLOPS: 100.6258000429082, Tokens per sec: 82223.93155359637, Loss: 2.2574267387390137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12809 , TFLOPS: 100.6365763904194, Tokens per sec: 82232.73718455563, Loss: 2.2750258445739746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12810 , TFLOPS: 101.17124942313568, Tokens per sec: 82669.63228329642, Loss: 2.242659091949463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12811 , TFLOPS: 99.93535313010787, Tokens per sec: 81659.7496074625, Loss: 2.2440571784973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12812 , TFLOPS: 100.65678266976802, Tokens per sec: 82249.24825556729, Loss: 2.2506959438323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12813 , TFLOPS: 96.96032355358527, Tokens per sec: 79228.77635641146, Loss: 2.248157501220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12814 , TFLOPS: 99.98884802995016, Tokens per sec: 81703.46166720486, Loss: 2.2270584106445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12815 , TFLOPS: 100.63079566499346, Tokens per sec: 82228.01360500099, Loss: 2.252117872238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12816 , TFLOPS: 100.54969671301909, Tokens per sec: 82161.74556366997, Loss: 2.2495322227478027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12817 , TFLOPS: 98.8412808651482, Tokens per sec: 80765.75499583832, Loss: 2.2540693283081055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12818 , TFLOPS: 101.14011015912884, Tokens per sec: 82644.18758908047, Loss: 2.2854256629943848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12819 , TFLOPS: 99.98321356062868, Tokens per sec: 81698.85759727813, Loss: 2.260331153869629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12820 , TFLOPS: 99.86570272104227, Tokens per sec: 81602.83646525396, Loss: 2.234112024307251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12821 , TFLOPS: 99.39396789508032, Tokens per sec: 81217.36979542571, Loss: 2.2760720252990723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12822 , TFLOPS: 100.63684175350872, Tokens per sec: 82232.95401956697, Loss: 2.2655422687530518 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12823 , TFLOPS: 101.32581678025952, Tokens per sec: 82795.93324972007, Loss: 2.268364191055298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12824 , TFLOPS: 98.77324876097413, Tokens per sec: 80710.16421221552, Loss: 2.241365671157837 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12825 , TFLOPS: 100.71800207088263, Tokens per sec: 82299.27220414563, Loss: 2.2576744556427 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12826 , TFLOPS: 100.09334430790024, Tokens per sec: 81788.84826589137, Loss: 2.2560901641845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12827 , TFLOPS: 99.42221682111736, Tokens per sec: 81240.45272008263, Loss: 2.2678005695343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12828 , TFLOPS: 99.93530323296379, Tokens per sec: 81659.70883522161, Loss: 2.252216339111328 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12829 , TFLOPS: 101.29281620303634, Tokens per sec: 82768.96762856057, Loss: 2.2531960010528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12830 , TFLOPS: 98.26138981163592, Tokens per sec: 80291.91108828971, Loss: 2.2460052967071533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12831 , TFLOPS: 100.46515482245832, Tokens per sec: 82092.6642085909, Loss: 2.2646353244781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12832 , TFLOPS: 99.57981635153067, Tokens per sec: 81369.23135335617, Loss: 2.2681124210357666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12833 , TFLOPS: 99.23778251710891, Tokens per sec: 81089.74670251596, Loss: 2.2514891624450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12834 , TFLOPS: 98.84508288025398, Tokens per sec: 80768.86172025382, Loss: 2.2498881816864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12835 , TFLOPS: 99.40792552562708, Tokens per sec: 81228.77493464685, Loss: 2.2604594230651855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12836 , TFLOPS: 98.84531161233085, Tokens per sec: 80769.04862312191, Loss: 2.2644667625427246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12837 , TFLOPS: 99.0896309404314, Tokens per sec: 80968.6882354519, Loss: 2.261137008666992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12838 , TFLOPS: 99.51691992705226, Tokens per sec: 81317.83706581686, Loss: 2.267859935760498 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12839 , TFLOPS: 98.61548122417913, Tokens per sec: 80581.24829660263, Loss: 2.2519967555999756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12840 , TFLOPS: 99.53979909544799, Tokens per sec: 81336.5322232752, Loss: 2.2732439041137695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12841 , TFLOPS: 99.6147783410631, Tokens per sec: 81397.7996950048, Loss: 2.259819984436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12842 , TFLOPS: 99.39080673164892, Tokens per sec: 81214.78672741033, Loss: 2.2532432079315186 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12843 , TFLOPS: 100.72092975255461, Tokens per sec: 82301.66448820503, Loss: 2.2614998817443848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12844 , TFLOPS: 99.03549666606052, Tokens per sec: 80924.45372632326, Loss: 2.2731235027313232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12845 , TFLOPS: 99.31489531825684, Tokens per sec: 81152.75755739404, Loss: 2.279913902282715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12846 , TFLOPS: 100.70903339368304, Tokens per sec: 82291.94366713155, Loss: 2.2353110313415527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12847 , TFLOPS: 100.69448984083255, Tokens per sec: 82280.05975571305, Loss: 2.2382266521453857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12848 , TFLOPS: 101.3360787616515, Tokens per sec: 82804.31857888235, Loss: 2.2517361640930176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12849 , TFLOPS: 99.50627531942544, Tokens per sec: 81309.13908290841, Loss: 2.270195960998535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12850 , TFLOPS: 100.76800961747449, Tokens per sec: 82340.1346577746, Loss: 2.2380969524383545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12851 , TFLOPS: 101.2831347570846, Tokens per sec: 82761.0566698508, Loss: 2.2711710929870605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12852 , TFLOPS: 100.15285654976346, Tokens per sec: 81837.47725069919, Loss: 2.250925064086914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12853 , TFLOPS: 99.99994324791906, Tokens per sec: 81712.52785542356, Loss: 2.2625954151153564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12854 , TFLOPS: 100.57001610482979, Tokens per sec: 82178.34906178618, Loss: 2.2603607177734375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12855 , TFLOPS: 99.54996661918312, Tokens per sec: 81344.84036865452, Loss: 2.2526726722717285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12856 , TFLOPS: 100.04789552790453, Tokens per sec: 81751.7108978012, Loss: 2.268984079360962 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12857 , TFLOPS: 100.74002227029094, Tokens per sec: 82317.26547593251, Loss: 2.2514092922210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12858 , TFLOPS: 100.11752648163363, Tokens per sec: 81808.60814255345, Loss: 2.2432210445404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12859 , TFLOPS: 99.4707642643716, Tokens per sec: 81280.12208568802, Loss: 2.255401372909546 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12860 , TFLOPS: 99.2607889824749, Tokens per sec: 81108.54587760562, Loss: 2.2426271438598633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12861 , TFLOPS: 101.32253585808938, Tokens per sec: 82793.25232375639, Loss: 2.254349708557129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12862 , TFLOPS: 99.20959776037024, Tokens per sec: 81066.71621224462, Loss: 2.2674012184143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12863 , TFLOPS: 100.68582732022053, Tokens per sec: 82272.98138712786, Loss: 2.2701663970947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12864 , TFLOPS: 99.99228488994598, Tokens per sec: 81706.27001398009, Loss: 2.264361619949341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12865 , TFLOPS: 98.7038567206938, Tokens per sec: 80653.46218979244, Loss: 2.241032838821411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12866 , TFLOPS: 99.96472230042367, Tokens per sec: 81683.74791255724, Loss: 2.2540249824523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12867 , TFLOPS: 100.1575694382141, Tokens per sec: 81841.32827317277, Loss: 2.2599194049835205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12868 , TFLOPS: 99.25376384373807, Tokens per sec: 81102.80545590058, Loss: 2.244844913482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12869 , TFLOPS: 99.45267101211087, Tokens per sec: 81265.33762350403, Loss: 2.2699451446533203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12870 , TFLOPS: 100.13419918518146, Tokens per sec: 81822.23183781597, Loss: 2.2510719299316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12871 , TFLOPS: 100.1124538315992, Tokens per sec: 81804.46314962869, Loss: 2.239588499069214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12872 , TFLOPS: 98.28201486471954, Tokens per sec: 80308.76435010043, Loss: 2.252138614654541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12873 , TFLOPS: 100.53422363638624, Tokens per sec: 82149.10211444086, Loss: 2.262843132019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12874 , TFLOPS: 99.50688146484507, Tokens per sec: 81309.63437993434, Loss: 2.258943557739258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12875 , TFLOPS: 98.94053058080065, Tokens per sec: 80846.85449341292, Loss: 2.25156307220459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12876 , TFLOPS: 100.15983874822975, Tokens per sec: 81843.18258480378, Loss: 2.267545223236084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12877 , TFLOPS: 98.35918214861444, Tokens per sec: 80371.81972423356, Loss: 2.2590079307556152 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12878 , TFLOPS: 100.8251157633288, Tokens per sec: 82386.79755959514, Loss: 2.241177558898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12879 , TFLOPS: 99.94597775021353, Tokens per sec: 81668.43125805288, Loss: 2.266568183898926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12880 , TFLOPS: 99.99839104624049, Tokens per sec: 81711.25951147478, Loss: 2.246540069580078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12881 , TFLOPS: 100.71195284713887, Tokens per sec: 82294.32922770373, Loss: 2.2443642616271973 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12882 , TFLOPS: 100.16234503476004, Tokens per sec: 81845.23053604524, Loss: 2.2559397220611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12883 , TFLOPS: 98.6591490784208, Tokens per sec: 80616.93042441407, Loss: 2.252004384994507 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12884 , TFLOPS: 100.60418696036677, Tokens per sec: 82206.27094748152, Loss: 2.2527167797088623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12885 , TFLOPS: 100.00978892700226, Tokens per sec: 81720.57301325277, Loss: 2.2471508979797363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12886 , TFLOPS: 101.38055720758726, Tokens per sec: 82840.66306203352, Loss: 2.247748851776123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12887 , TFLOPS: 98.6776972206696, Tokens per sec: 80632.08658891721, Loss: 2.241903305053711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12888 , TFLOPS: 99.91583096178256, Tokens per sec: 81643.79754117856, Loss: 2.281235694885254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12889 , TFLOPS: 101.28444250843819, Tokens per sec: 82762.12526714634, Loss: 2.2692058086395264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12890 , TFLOPS: 100.73446623405849, Tokens per sec: 82312.72549570189, Loss: 2.258805990219116 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12891 , TFLOPS: 99.46038344901038, Tokens per sec: 81271.63965423041, Loss: 2.2762646675109863 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12892 , TFLOPS: 100.67839163678947, Tokens per sec: 82266.90549878482, Loss: 2.2546446323394775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12893 , TFLOPS: 99.47453597789091, Tokens per sec: 81283.2040498972, Loss: 2.257728099822998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12894 , TFLOPS: 100.00921882983421, Tokens per sec: 81720.10717218116, Loss: 2.2487142086029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12895 , TFLOPS: 100.78399055190091, Tokens per sec: 82353.19309068029, Loss: 2.2321767807006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12896 , TFLOPS: 99.39000150660226, Tokens per sec: 81214.12875729636, Loss: 2.2435057163238525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12897 , TFLOPS: 99.41953361982881, Tokens per sec: 81238.260207238, Loss: 2.25396728515625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12898 , TFLOPS: 100.0050398067324, Tokens per sec: 81716.69238482704, Loss: 2.266200065612793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12899 , TFLOPS: 100.74567381500714, Tokens per sec: 82321.88349860383, Loss: 2.270927667617798 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12900 , TFLOPS: 100.08666349955433, Tokens per sec: 81783.38920541263, Loss: 2.2506415843963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12901 , TFLOPS: 99.6539317074884, Tokens per sec: 81429.79291860822, Loss: 2.261662721633911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12902 , TFLOPS: 99.31924525283439, Tokens per sec: 81156.31200091462, Loss: 2.283754348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12903 , TFLOPS: 99.4824025354536, Tokens per sec: 81289.63201658489, Loss: 2.2333133220672607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12904 , TFLOPS: 99.55426102422837, Tokens per sec: 81348.3494375648, Loss: 2.24176287651062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12905 , TFLOPS: 100.75694563175917, Tokens per sec: 82331.0939902343, Loss: 2.279700756072998 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12906 , TFLOPS: 98.88764558056057, Tokens per sec: 80803.6407983357, Loss: 2.2464780807495117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12907 , TFLOPS: 99.59143565539074, Tokens per sec: 81378.72578564772, Loss: 2.252061605453491 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12908 , TFLOPS: 100.1476233880772, Tokens per sec: 81833.2010995718, Loss: 2.2548584938049316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12909 , TFLOPS: 100.7110736674766, Tokens per sec: 82293.61082736959, Loss: 2.2679216861724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12910 , TFLOPS: 98.66879094497538, Tokens per sec: 80624.80904177955, Loss: 2.2581162452697754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12911 , TFLOPS: 100.5102281072757, Tokens per sec: 82129.49474990476, Loss: 2.267777442932129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12912 , TFLOPS: 100.19902208539973, Tokens per sec: 81875.20029827423, Loss: 2.236983060836792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12913 , TFLOPS: 100.17129136299168, Tokens per sec: 81852.54081114229, Loss: 2.2451179027557373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12914 , TFLOPS: 99.43298065598975, Tokens per sec: 81249.2481266426, Loss: 2.2466063499450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12915 , TFLOPS: 99.06407053958917, Tokens per sec: 80947.80217394046, Loss: 2.25534725189209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12916 , TFLOPS: 99.35567708389804, Tokens per sec: 81186.08138791549, Loss: 2.261178493499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12917 , TFLOPS: 100.11338319647724, Tokens per sec: 81805.22255759452, Loss: 2.279473304748535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12918 , TFLOPS: 100.75379484545802, Tokens per sec: 82328.51940163916, Loss: 2.2627182006835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12919 , TFLOPS: 100.17423627652813, Tokens per sec: 81854.94717580175, Loss: 2.262021780014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12920 , TFLOPS: 100.76744070875262, Tokens per sec: 82339.66978781295, Loss: 2.2613718509674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12921 , TFLOPS: 98.58736263612847, Tokens per sec: 80558.27187446957, Loss: 2.273399591445923 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12922 , TFLOPS: 100.78301354739887, Tokens per sec: 82352.39475515133, Loss: 2.239597797393799 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12923 , TFLOPS: 100.0841907070583, Tokens per sec: 81781.36862300878, Loss: 2.264436721801758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12924 , TFLOPS: 101.291610929185, Tokens per sec: 82767.98276827013, Loss: 2.250190019607544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12925 , TFLOPS: 98.68740047651026, Tokens per sec: 80640.01536904865, Loss: 2.2502317428588867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12926 , TFLOPS: 100.09819023810203, Tokens per sec: 81792.80800020459, Loss: 2.248929977416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12927 , TFLOPS: 100.66391773282805, Tokens per sec: 82255.07849926651, Loss: 2.2349095344543457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12928 , TFLOPS: 99.94714606789198, Tokens per sec: 81669.3859205031, Loss: 2.261047840118408 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12929 , TFLOPS: 99.52638349533572, Tokens per sec: 81325.56999107519, Loss: 2.2609663009643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12930 , TFLOPS: 100.69342427594943, Tokens per sec: 82279.18905521698, Loss: 2.2487754821777344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12931 , TFLOPS: 99.44009493543055, Tokens per sec: 81255.06138751154, Loss: 2.2584400177001953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12932 , TFLOPS: 100.10579060648745, Tokens per sec: 81799.0184568632, Loss: 2.270756959915161 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12933 , TFLOPS: 100.36436254759592, Tokens per sec: 82010.30424617686, Loss: 2.222541093826294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12934 , TFLOPS: 100.04392900273268, Tokens per sec: 81748.46974797583, Loss: 2.2607100009918213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12935 , TFLOPS: 99.49383574397409, Tokens per sec: 81298.97438558395, Loss: 2.254359245300293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12936 , TFLOPS: 100.02644044456791, Tokens per sec: 81734.17939690387, Loss: 2.250584602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12937 , TFLOPS: 99.89231136590504, Tokens per sec: 81624.57907393879, Loss: 2.2393486499786377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12938 , TFLOPS: 100.1587756155153, Tokens per sec: 81842.31387169534, Loss: 2.2552380561828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12939 , TFLOPS: 100.12204317703764, Tokens per sec: 81812.29885063817, Loss: 2.258643865585327 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12940 , TFLOPS: 99.90944446417137, Tokens per sec: 81638.57896957739, Loss: 2.250300884246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12941 , TFLOPS: 99.20061642792547, Tokens per sec: 81059.3773343039, Loss: 2.2512006759643555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12942 , TFLOPS: 99.24341231410689, Tokens per sec: 81094.34695456688, Loss: 2.2461609840393066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12943 , TFLOPS: 99.97641706538141, Tokens per sec: 81693.30400605424, Loss: 2.2620906829833984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12944 , TFLOPS: 99.09424137000553, Tokens per sec: 80972.45553613994, Loss: 2.262559413909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12945 , TFLOPS: 100.14006317712982, Tokens per sec: 81827.02345658954, Loss: 2.2619998455047607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12946 , TFLOPS: 100.18891973687538, Tokens per sec: 81866.94540923738, Loss: 2.254420518875122 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12947 , TFLOPS: 100.6024612970291, Tokens per sec: 82204.8608635458, Loss: 2.2448995113372803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12948 , TFLOPS: 98.80784461717712, Tokens per sec: 80738.43337689555, Loss: 2.2368416786193848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12949 , TFLOPS: 100.67297212455354, Tokens per sec: 82262.47707582619, Loss: 2.235604763031006 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12950 , TFLOPS: 100.0281110635877, Tokens per sec: 81735.5445027105, Loss: 2.2604243755340576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12951 , TFLOPS: 100.03488562089161, Tokens per sec: 81741.08016787615, Loss: 2.2490005493164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12952 , TFLOPS: 99.44393663636939, Tokens per sec: 81258.20054024283, Loss: 2.2694129943847656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12953 , TFLOPS: 98.90427510758995, Tokens per sec: 80817.22921295353, Loss: 2.2702345848083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12954 , TFLOPS: 100.11579954473106, Tokens per sec: 81807.19701795506, Loss: 2.2525765895843506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12955 , TFLOPS: 99.47423802528297, Tokens per sec: 81282.96058515128, Loss: 2.2462832927703857 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12956 , TFLOPS: 100.72724226762757, Tokens per sec: 82306.82260676975, Loss: 2.2702579498291016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12957 , TFLOPS: 100.67504056816358, Tokens per sec: 82264.16725434642, Loss: 2.2555603981018066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12958 , TFLOPS: 98.6361900700285, Tokens per sec: 80598.17002763927, Loss: 2.222872257232666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12959 , TFLOPS: 100.03222720741913, Tokens per sec: 81738.90790979411, Loss: 2.251338243484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12960 , TFLOPS: 100.07107075755675, Tokens per sec: 81770.64797453351, Loss: 2.245084524154663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12961 , TFLOPS: 100.19836975498923, Tokens per sec: 81874.66726230331, Loss: 2.255377769470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12962 , TFLOPS: 101.29763396780304, Tokens per sec: 82772.90434817174, Loss: 2.2599575519561768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12963 , TFLOPS: 98.7901867194064, Tokens per sec: 80724.00465407237, Loss: 2.2653164863586426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12964 , TFLOPS: 100.19839458496502, Tokens per sec: 81874.68755151572, Loss: 2.2697248458862305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12965 , TFLOPS: 100.59993802354178, Tokens per sec: 82202.79903182446, Loss: 2.2527778148651123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12966 , TFLOPS: 100.19653510300391, Tokens per sec: 81873.16812093796, Loss: 2.2584874629974365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12967 , TFLOPS: 99.46273202595347, Tokens per sec: 81273.55873690837, Loss: 2.2568843364715576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12968 , TFLOPS: 100.02785951970847, Tokens per sec: 81735.33895973147, Loss: 2.2694997787475586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12969 , TFLOPS: 98.49802226932562, Tokens per sec: 80485.26956092933, Loss: 2.251936912536621 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12970 , TFLOPS: 99.8666317527175, Tokens per sec: 81603.59560095119, Loss: 2.276848793029785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12971 , TFLOPS: 100.0319653848647, Tokens per sec: 81738.693967845, Loss: 2.248432159423828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12972 , TFLOPS: 100.00740382391496, Tokens per sec: 81718.62408412212, Loss: 2.2494187355041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12973 , TFLOPS: 99.56820931662585, Tokens per sec: 81359.74694634377, Loss: 2.2461090087890625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12974 , TFLOPS: 99.4516226968694, Tokens per sec: 81264.48101813419, Loss: 2.267881155014038 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12975 , TFLOPS: 101.27337480405774, Tokens per sec: 82753.08156098903, Loss: 2.2514493465423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12976 , TFLOPS: 99.5785076106219, Tokens per sec: 81368.16194746962, Loss: 2.2727229595184326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12977 , TFLOPS: 100.22836690033661, Tokens per sec: 81899.17870196188, Loss: 2.2618229389190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12978 , TFLOPS: 100.11158123615533, Tokens per sec: 81803.7501294289, Loss: 2.2456822395324707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12979 , TFLOPS: 99.30152548039631, Tokens per sec: 81141.83271870794, Loss: 2.2754809856414795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12980 , TFLOPS: 100.0690193923945, Tokens per sec: 81768.97175125258, Loss: 2.2541403770446777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12981 , TFLOPS: 100.71600304825172, Tokens per sec: 82297.6387512945, Loss: 2.2672624588012695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12982 , TFLOPS: 99.47297372349364, Tokens per sec: 81281.92749161318, Loss: 2.2525243759155273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12983 , TFLOPS: 100.70842617063192, Tokens per sec: 82291.44748954517, Loss: 2.243147373199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12984 , TFLOPS: 100.11988430787333, Tokens per sec: 81810.53478306977, Loss: 2.259032726287842 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12985 , TFLOPS: 99.86565076295179, Tokens per sec: 81602.7940089607, Loss: 2.2599964141845703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12986 , TFLOPS: 98.8804834667472, Tokens per sec: 80797.78845076959, Loss: 2.2554609775543213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12987 , TFLOPS: 100.64210569226779, Tokens per sec: 82237.25531943285, Loss: 2.2553694248199463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12988 , TFLOPS: 100.159946239308, Tokens per sec: 81843.2704185309, Loss: 2.239243745803833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12989 , TFLOPS: 100.14515682685976, Tokens per sec: 81831.1856089061, Loss: 2.2715892791748047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12990 , TFLOPS: 98.90032983767682, Tokens per sec: 80814.00543134722, Loss: 2.2731244564056396 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12991 , TFLOPS: 98.95498545920209, Tokens per sec: 80858.6659466564, Loss: 2.2577342987060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12992 , TFLOPS: 100.76189918877722, Tokens per sec: 82335.14166918964, Loss: 2.27567982673645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12993 , TFLOPS: 99.46885835552753, Tokens per sec: 81278.56471850908, Loss: 2.249966621398926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12994 , TFLOPS: 100.74579658163474, Tokens per sec: 82321.98381437553, Loss: 2.235085964202881 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12995 , TFLOPS: 100.67996334771438, Tokens per sec: 82268.189784241, Loss: 2.2310023307800293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12996 , TFLOPS: 100.0247824371998, Tokens per sec: 81732.82459640247, Loss: 2.246441602706909 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12997 , TFLOPS: 99.31903628548494, Tokens per sec: 81156.14124831409, Loss: 2.2431411743164062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12998 , TFLOPS: 100.73614019243976, Tokens per sec: 82314.09333018676, Loss: 2.2405261993408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12999 , TFLOPS: 99.36044678210861, Tokens per sec: 81189.9788311063, Loss: 2.255889892578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13000 , TFLOPS: 101.22890129875299, Tokens per sec: 82716.74111495464, Loss: 2.268414258956909 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/optimizer.pt +[rank0]:[2024-08-29 13:43:01,754] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007703242008574307, 'preprocessing_with_comm': 0.0016454790020361543, 'state_converting': 2.353409224000643, : 2.3643941909977}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013000, took 14.48s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011000 + eval ppl=7.457808494567871, eval loss=2.0092616081237793 +------------------------------------------------------------------ +iteration: 13001 , TFLOPS: 97.83182702363281, Tokens per sec: 79940.90427628247, Loss: 2.2601263523101807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13002 , TFLOPS: 98.40076500891983, Tokens per sec: 80405.79814982715, Loss: 2.2599241733551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13003 , TFLOPS: 100.61958515187136, Tokens per sec: 82218.85320614465, Loss: 2.2543275356292725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13004 , TFLOPS: 98.90910960295894, Tokens per sec: 80821.1796035705, Loss: 2.266089677810669 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13005 , TFLOPS: 99.36395352454306, Tokens per sec: 81192.84428062108, Loss: 2.2540316581726074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13006 , TFLOPS: 100.74921291857606, Tokens per sec: 82324.77539123462, Loss: 2.2547178268432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13007 , TFLOPS: 100.81319775687571, Tokens per sec: 82377.05904972555, Loss: 2.2533557415008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13008 , TFLOPS: 100.04439206526878, Tokens per sec: 81748.84812829437, Loss: 2.2569384574890137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13009 , TFLOPS: 101.35442521928366, Tokens per sec: 82819.30994169343, Loss: 2.276552677154541 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13010 , TFLOPS: 100.03359751091676, Tokens per sec: 81740.02762005679, Loss: 2.2416648864746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13011 , TFLOPS: 100.74884975340274, Tokens per sec: 82324.4786396228, Loss: 2.236255645751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13012 , TFLOPS: 100.67229987329476, Tokens per sec: 82261.92776201735, Loss: 2.2867090702056885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13013 , TFLOPS: 100.31038842592055, Tokens per sec: 81966.20060193841, Loss: 2.2572100162506104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13014 , TFLOPS: 99.26047676573782, Tokens per sec: 81108.29075727257, Loss: 2.262850761413574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13015 , TFLOPS: 99.99361291815899, Tokens per sec: 81707.35518001943, Loss: 2.2731149196624756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13016 , TFLOPS: 100.62382435338097, Tokens per sec: 82222.3171668249, Loss: 2.253291606903076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13017 , TFLOPS: 99.49912996255493, Tokens per sec: 81303.30042787165, Loss: 2.2565321922302246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13018 , TFLOPS: 100.54516134195877, Tokens per sec: 82158.03959522577, Loss: 2.249800205230713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13019 , TFLOPS: 98.1348483934105, Tokens per sec: 80188.51083799181, Loss: 2.2467167377471924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13020 , TFLOPS: 101.24554031702795, Tokens per sec: 82730.33728511352, Loss: 2.262282133102417 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13021 , TFLOPS: 99.62981199573507, Tokens per sec: 81410.08408123795, Loss: 2.262688159942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13022 , TFLOPS: 98.61753585821444, Tokens per sec: 80582.92719096386, Loss: 2.2500081062316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13023 , TFLOPS: 100.6989079387278, Tokens per sec: 82283.66989723523, Loss: 2.2718281745910645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13024 , TFLOPS: 98.60232556804561, Tokens per sec: 80570.49847131921, Loss: 2.2335567474365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13025 , TFLOPS: 100.73441502385198, Tokens per sec: 82312.68365052389, Loss: 2.259861707687378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13026 , TFLOPS: 99.56950687023055, Tokens per sec: 81360.80721079619, Loss: 2.249974489212036 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13027 , TFLOPS: 97.8192431067125, Tokens per sec: 79930.62163382804, Loss: 2.2726879119873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13028 , TFLOPS: 101.28129950124118, Tokens per sec: 82759.55703505746, Loss: 2.243889808654785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13029 , TFLOPS: 99.92177784665665, Tokens per sec: 81648.65689389562, Loss: 2.2457375526428223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13030 , TFLOPS: 98.84269148949107, Tokens per sec: 80766.90765330156, Loss: 2.261849880218506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13031 , TFLOPS: 99.76371260876473, Tokens per sec: 81519.49771905293, Loss: 2.23154616355896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13032 , TFLOPS: 100.72516115789968, Tokens per sec: 82305.12207843855, Loss: 2.257023811340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13033 , TFLOPS: 98.22389622627776, Tokens per sec: 80261.27411452279, Loss: 2.241497755050659 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13034 , TFLOPS: 100.09785876540239, Tokens per sec: 81792.53714532885, Loss: 2.2352044582366943 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13035 , TFLOPS: 98.67833063468936, Tokens per sec: 80632.6041678183, Loss: 2.2453994750976562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13036 , TFLOPS: 99.32659337062034, Tokens per sec: 81162.31633711491, Loss: 2.271054267883301 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13037 , TFLOPS: 97.92152378861947, Tokens per sec: 80014.1978119532, Loss: 2.249746322631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13038 , TFLOPS: 100.74648137670901, Tokens per sec: 82322.54337805891, Loss: 2.2551677227020264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13039 , TFLOPS: 98.99623082090005, Tokens per sec: 80892.36859344985, Loss: 2.2765257358551025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13040 , TFLOPS: 97.86062517576576, Tokens per sec: 79964.43598772069, Loss: 2.2474136352539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13041 , TFLOPS: 100.5177045348864, Tokens per sec: 82135.60393136583, Loss: 2.249964714050293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13042 , TFLOPS: 99.09695700399281, Tokens per sec: 80974.67455057759, Loss: 2.2462315559387207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13043 , TFLOPS: 99.58966808179973, Tokens per sec: 81377.28145576511, Loss: 2.262883424758911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13044 , TFLOPS: 100.72678364872756, Tokens per sec: 82306.44785746065, Loss: 2.2810797691345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13045 , TFLOPS: 100.84428210720681, Tokens per sec: 82402.45887256344, Loss: 2.2685065269470215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13046 , TFLOPS: 99.24364673947237, Tokens per sec: 81094.53850956765, Loss: 2.2646334171295166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13047 , TFLOPS: 101.31675933880868, Tokens per sec: 82788.5321811513, Loss: 2.2729547023773193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13048 , TFLOPS: 100.66077594245225, Tokens per sec: 82252.51126147357, Loss: 2.2595698833465576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13049 , TFLOPS: 99.97096321791359, Tokens per sec: 81688.84752689375, Loss: 2.2555997371673584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13050 , TFLOPS: 100.53531507463988, Tokens per sec: 82149.99395673403, Loss: 2.23876690864563 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13051 , TFLOPS: 100.70031801618488, Tokens per sec: 82284.82210782403, Loss: 2.2550415992736816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13052 , TFLOPS: 98.70719202965981, Tokens per sec: 80656.18755660705, Loss: 2.2597763538360596 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13053 , TFLOPS: 100.69896356517192, Tokens per sec: 82283.71535103467, Loss: 2.2717275619506836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13054 , TFLOPS: 100.6911036907656, Tokens per sec: 82277.2928453261, Loss: 2.237121820449829 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13055 , TFLOPS: 99.42841270455085, Tokens per sec: 81245.51553593235, Loss: 2.2620840072631836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13056 , TFLOPS: 100.65120771508397, Tokens per sec: 82244.69281658286, Loss: 2.2593321800231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13057 , TFLOPS: 95.62695605528546, Tokens per sec: 78139.24744961775, Loss: 2.2462198734283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13058 , TFLOPS: 101.35423502150122, Tokens per sec: 82819.15452618926, Loss: 2.239370822906494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13059 , TFLOPS: 99.48634988229563, Tokens per sec: 81292.85749530324, Loss: 2.247734785079956 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13060 , TFLOPS: 98.73946829451411, Tokens per sec: 80682.56132348446, Loss: 2.242009401321411 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13061 , TFLOPS: 100.5787346091063, Tokens per sec: 82185.47317606478, Loss: 2.2501161098480225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13062 , TFLOPS: 98.41297478177921, Tokens per sec: 80415.77506953807, Loss: 2.2525253295898438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13063 , TFLOPS: 101.30365402852337, Tokens per sec: 82777.82349475648, Loss: 2.2465596199035645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13064 , TFLOPS: 98.97147389689268, Tokens per sec: 80872.13907354351, Loss: 2.2705671787261963 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13065 , TFLOPS: 98.27014424128164, Tokens per sec: 80299.0645581123, Loss: 2.2740697860717773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13066 , TFLOPS: 100.55463736846029, Tokens per sec: 82165.78270041477, Loss: 2.2631912231445312 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13067 , TFLOPS: 100.66755369442124, Tokens per sec: 82258.04953708229, Loss: 2.239121913909912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13068 , TFLOPS: 98.88508397273175, Tokens per sec: 80801.54764263713, Loss: 2.2480461597442627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13069 , TFLOPS: 100.01194265209864, Tokens per sec: 81722.33287747085, Loss: 2.25868821144104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13070 , TFLOPS: 100.73544141900341, Tokens per sec: 82313.52234442387, Loss: 2.2798938751220703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13071 , TFLOPS: 98.16189573091538, Tokens per sec: 80210.61191372745, Loss: 2.2491397857666016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13072 , TFLOPS: 100.1165428914003, Tokens per sec: 81807.80442565393, Loss: 2.2399954795837402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13073 , TFLOPS: 98.94105021568262, Tokens per sec: 80847.27910045156, Loss: 2.2383663654327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13074 , TFLOPS: 99.41018031860807, Tokens per sec: 81230.61738403514, Loss: 2.255378246307373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13075 , TFLOPS: 98.29416903834746, Tokens per sec: 80318.69583824807, Loss: 2.2798469066619873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13076 , TFLOPS: 100.14054056428039, Tokens per sec: 81827.41354191932, Loss: 2.259150743484497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13077 , TFLOPS: 98.92877425641313, Tokens per sec: 80837.24809812116, Loss: 2.2520298957824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13078 , TFLOPS: 97.22823252215731, Tokens per sec: 79447.69167122207, Loss: 2.2476582527160645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13079 , TFLOPS: 100.67796611883573, Tokens per sec: 82266.55779711102, Loss: 2.2684531211853027 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13080 , TFLOPS: 99.0502744925344, Tokens per sec: 80936.52906875017, Loss: 2.251955986022949 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13081 , TFLOPS: 99.75815622220317, Tokens per sec: 81514.95745255935, Loss: 2.2442610263824463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13082 , TFLOPS: 100.71844809411937, Tokens per sec: 82299.63666121403, Loss: 2.2455151081085205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13083 , TFLOPS: 100.87339541501261, Tokens per sec: 82426.24810581478, Loss: 2.2566332817077637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13084 , TFLOPS: 100.1560635879777, Tokens per sec: 81840.09780418056, Loss: 2.246215581893921 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13085 , TFLOPS: 101.31270018992893, Tokens per sec: 82785.21534610986, Loss: 2.2486684322357178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13086 , TFLOPS: 100.77793860499088, Tokens per sec: 82348.24788906911, Loss: 2.253056287765503 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13087 , TFLOPS: 99.3444951641638, Tokens per sec: 81176.94435345243, Loss: 2.241123676300049 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13088 , TFLOPS: 100.63085223806578, Tokens per sec: 82228.05983231471, Loss: 2.2734944820404053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13089 , TFLOPS: 100.69697232709994, Tokens per sec: 82282.08825914703, Loss: 2.253206968307495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13090 , TFLOPS: 99.43111794836352, Tokens per sec: 81247.72606029086, Loss: 2.244344711303711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13091 , TFLOPS: 100.50408860096202, Tokens per sec: 82124.4780012509, Loss: 2.2274281978607178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13092 , TFLOPS: 99.41262726018468, Tokens per sec: 81232.61684298726, Loss: 2.249164581298828 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13093 , TFLOPS: 99.15892896005192, Tokens per sec: 81025.31343117355, Loss: 2.244082450866699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13094 , TFLOPS: 101.26438758652326, Tokens per sec: 82745.73787419002, Loss: 2.259390354156494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13095 , TFLOPS: 98.3558378356999, Tokens per sec: 80369.08700006081, Loss: 2.2644872665405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13096 , TFLOPS: 100.6627740544212, Tokens per sec: 82254.14397019937, Loss: 2.236098289489746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13097 , TFLOPS: 99.40920566624243, Tokens per sec: 81229.82097049739, Loss: 2.259608507156372 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13098 , TFLOPS: 99.35540993156673, Tokens per sec: 81185.86309086847, Loss: 2.2532424926757812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13099 , TFLOPS: 100.03019106364454, Tokens per sec: 81737.2441243009, Loss: 2.2517170906066895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13100 , TFLOPS: 98.53568834054636, Tokens per sec: 80516.04747733472, Loss: 2.264450788497925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13101 , TFLOPS: 99.87333480505968, Tokens per sec: 81609.0728375719, Loss: 2.2408249378204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13102 , TFLOPS: 100.15259261466196, Tokens per sec: 81837.26158253348, Loss: 2.258990526199341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13103 , TFLOPS: 97.63442982538378, Tokens per sec: 79779.60594413723, Loss: 2.256632089614868 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13104 , TFLOPS: 100.68007003883118, Tokens per sec: 82268.276964299, Loss: 2.28364896774292 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13105 , TFLOPS: 100.57262791167565, Tokens per sec: 82180.48323639383, Loss: 2.2539713382720947 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13106 , TFLOPS: 97.71486568103956, Tokens per sec: 79845.3321523967, Loss: 2.2510831356048584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13107 , TFLOPS: 100.56552804957995, Tokens per sec: 82174.68175630872, Loss: 2.250135898590088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13108 , TFLOPS: 99.91268052215062, Tokens per sec: 81641.22323585578, Loss: 2.254737138748169 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13109 , TFLOPS: 98.76963723436965, Tokens per sec: 80707.21314085803, Loss: 2.2751824855804443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13110 , TFLOPS: 98.88854533403425, Tokens per sec: 80804.37601006078, Loss: 2.274714946746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13111 , TFLOPS: 100.15981696769698, Tokens per sec: 81843.16478736978, Loss: 2.2461295127868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13112 , TFLOPS: 98.17592408361115, Tokens per sec: 80222.07484183711, Loss: 2.266911506652832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13113 , TFLOPS: 98.76266358905741, Tokens per sec: 80701.5147957558, Loss: 2.2652812004089355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13114 , TFLOPS: 100.78214928297561, Tokens per sec: 82351.68854244295, Loss: 2.2534422874450684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13115 , TFLOPS: 99.44912018463765, Tokens per sec: 81262.43615096924, Loss: 2.253288984298706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13116 , TFLOPS: 98.48312726014733, Tokens per sec: 80473.0984654981, Loss: 2.2658729553222656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13117 , TFLOPS: 100.62838134189465, Tokens per sec: 82226.04079944675, Loss: 2.2835288047790527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13118 , TFLOPS: 99.57597059210562, Tokens per sec: 81366.0888843313, Loss: 2.255528211593628 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13119 , TFLOPS: 99.48557606220517, Tokens per sec: 81292.22518698743, Loss: 2.2596852779388428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13120 , TFLOPS: 99.51580070116025, Tokens per sec: 81316.92251752905, Loss: 2.2504143714904785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13121 , TFLOPS: 100.76893717900427, Tokens per sec: 82340.89259217815, Loss: 2.2550058364868164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13122 , TFLOPS: 99.46471605077905, Tokens per sec: 81275.17993466668, Loss: 2.246636390686035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13123 , TFLOPS: 101.33499530903545, Tokens per sec: 82803.43326185922, Loss: 2.2575807571411133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13124 , TFLOPS: 100.06349129374516, Tokens per sec: 81764.45459954032, Loss: 2.2739036083221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13125 , TFLOPS: 100.06906298688251, Tokens per sec: 81769.00737343097, Loss: 2.244762420654297 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13126 , TFLOPS: 101.22632203574972, Tokens per sec: 82714.63353275853, Loss: 2.2788174152374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13127 , TFLOPS: 100.07527054555783, Tokens per sec: 81774.07972942136, Loss: 2.2521374225616455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13128 , TFLOPS: 98.60142207810979, Tokens per sec: 80569.76020643476, Loss: 2.266728639602661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13129 , TFLOPS: 100.75481131732428, Tokens per sec: 82329.3499869674, Loss: 2.2441554069519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13130 , TFLOPS: 100.06415010702986, Tokens per sec: 81764.99293283459, Loss: 2.2411184310913086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13131 , TFLOPS: 99.41430948928371, Tokens per sec: 81233.99143568851, Loss: 2.2531800270080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13132 , TFLOPS: 101.357608175114, Tokens per sec: 82821.91081683896, Loss: 2.2646868228912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13133 , TFLOPS: 97.71730570444154, Tokens per sec: 79847.32595833026, Loss: 2.2330551147460938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13134 , TFLOPS: 101.22567556534979, Tokens per sec: 82714.10528515311, Loss: 2.2493906021118164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13135 , TFLOPS: 98.7906634373914, Tokens per sec: 80724.39419260972, Loss: 2.2461788654327393 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13136 , TFLOPS: 100.0465684950011, Tokens per sec: 81750.62654505493, Loss: 2.2746403217315674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13137 , TFLOPS: 99.36454889793782, Tokens per sec: 81193.33077554821, Loss: 2.2858235836029053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13138 , TFLOPS: 98.62046669399798, Tokens per sec: 80585.32205232902, Loss: 2.2521607875823975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13139 , TFLOPS: 100.69037915414708, Tokens per sec: 82276.70080780388, Loss: 2.264486074447632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13140 , TFLOPS: 98.82586770029367, Tokens per sec: 80753.16050206551, Loss: 2.2491793632507324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13141 , TFLOPS: 98.48604628236593, Tokens per sec: 80475.48367369524, Loss: 2.2546284198760986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13142 , TFLOPS: 100.67827943806037, Tokens per sec: 82266.81381831503, Loss: 2.2600789070129395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13143 , TFLOPS: 100.06898269565376, Tokens per sec: 81768.94176540108, Loss: 2.246025562286377 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13144 , TFLOPS: 98.810955707898, Tokens per sec: 80740.97552921019, Loss: 2.2482595443725586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13145 , TFLOPS: 100.50954760225925, Tokens per sec: 82128.93869173806, Loss: 2.2510476112365723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13146 , TFLOPS: 100.81014391182453, Tokens per sec: 82374.56367432125, Loss: 2.2682597637176514 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13147 , TFLOPS: 99.43335976632073, Tokens per sec: 81249.55790745323, Loss: 2.2477104663848877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13148 , TFLOPS: 99.33998306365223, Tokens per sec: 81173.25739997262, Loss: 2.2439262866973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13149 , TFLOPS: 100.11100590824648, Tokens per sec: 81803.28001418433, Loss: 2.2465291023254395 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13150 , TFLOPS: 98.6962691857946, Tokens per sec: 80647.26221970578, Loss: 2.2889585494995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13151 , TFLOPS: 98.852396890247, Tokens per sec: 80774.83818609848, Loss: 2.2303853034973145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13152 , TFLOPS: 100.12169859783577, Tokens per sec: 81812.01728610205, Loss: 2.2398157119750977 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13153 , TFLOPS: 98.8374496466714, Tokens per sec: 80762.62440859659, Loss: 2.253321647644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13154 , TFLOPS: 98.44977255324588, Tokens per sec: 80445.84347586239, Loss: 2.2578155994415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13155 , TFLOPS: 100.13964673174591, Tokens per sec: 81826.68316834609, Loss: 2.2615551948547363 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13156 , TFLOPS: 99.61986865184114, Tokens per sec: 81401.95911897779, Loss: 2.2508366107940674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13157 , TFLOPS: 99.64996959175532, Tokens per sec: 81426.5553718488, Loss: 2.2459640502929688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13158 , TFLOPS: 100.09383746070795, Tokens per sec: 81789.25123374542, Loss: 2.2732887268066406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13159 , TFLOPS: 100.75262492255088, Tokens per sec: 82327.56342751524, Loss: 2.23752498626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13160 , TFLOPS: 100.13395629964036, Tokens per sec: 81822.0333697879, Loss: 2.277366876602173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13161 , TFLOPS: 101.30768323437765, Tokens per sec: 82781.11586258098, Loss: 2.2736668586730957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13162 , TFLOPS: 99.80557711244236, Tokens per sec: 81553.7062826961, Loss: 2.245047092437744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13163 , TFLOPS: 100.09931702337825, Tokens per sec: 81793.72872545983, Loss: 2.2500498294830322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13164 , TFLOPS: 100.64658261749814, Tokens per sec: 82240.91353028487, Loss: 2.2412755489349365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13165 , TFLOPS: 100.08996094383423, Tokens per sec: 81786.08363201749, Loss: 2.251892566680908 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13166 , TFLOPS: 99.42467915582345, Tokens per sec: 81242.46475715711, Loss: 2.2733583450317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13167 , TFLOPS: 100.69592410770355, Tokens per sec: 82281.23173209466, Loss: 2.2707302570343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13168 , TFLOPS: 99.34633290433979, Tokens per sec: 81178.44601825788, Loss: 2.2345170974731445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13169 , TFLOPS: 99.3960150216594, Tokens per sec: 81219.04255525119, Loss: 2.253195285797119 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13170 , TFLOPS: 101.19281155876683, Tokens per sec: 82687.25125937938, Loss: 2.2556543350219727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13171 , TFLOPS: 98.25894689422967, Tokens per sec: 80289.91491759075, Loss: 2.2771224975585938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13172 , TFLOPS: 100.5608690407679, Tokens per sec: 82170.87476027483, Loss: 2.266676425933838 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13173 , TFLOPS: 98.73904453274096, Tokens per sec: 80682.21505683103, Loss: 2.2630386352539062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13174 , TFLOPS: 100.09092136264485, Tokens per sec: 81786.86841495104, Loss: 2.263582944869995 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13175 , TFLOPS: 99.41866238750806, Tokens per sec: 81237.5483008812, Loss: 2.2443511486053467 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13176 , TFLOPS: 98.37486005654004, Tokens per sec: 80384.63054638483, Loss: 2.2358977794647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13177 , TFLOPS: 100.63400683785454, Tokens per sec: 82230.63753700872, Loss: 2.252479314804077 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13178 , TFLOPS: 98.90962272957951, Tokens per sec: 80821.59889254124, Loss: 2.2475264072418213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13179 , TFLOPS: 98.42982206149676, Tokens per sec: 80429.54141548285, Loss: 2.2507150173187256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13180 , TFLOPS: 100.71069394895983, Tokens per sec: 82293.30054959471, Loss: 2.234055280685425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13181 , TFLOPS: 99.96607459160585, Tokens per sec: 81684.85290449327, Loss: 2.2581441402435303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13182 , TFLOPS: 99.0449795727272, Tokens per sec: 80932.20245347235, Loss: 2.2476813793182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13183 , TFLOPS: 100.68746088174821, Tokens per sec: 82274.31621230373, Loss: 2.2592504024505615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13184 , TFLOPS: 100.65661891474682, Tokens per sec: 82249.11444712404, Loss: 2.2462427616119385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13185 , TFLOPS: 98.64241546198839, Tokens per sec: 80603.25695566554, Loss: 2.263179302215576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13186 , TFLOPS: 99.97795446589384, Tokens per sec: 81694.56025558915, Loss: 2.2602837085723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13187 , TFLOPS: 99.55543494865374, Tokens per sec: 81349.30868143229, Loss: 2.2687885761260986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13188 , TFLOPS: 98.85641076562317, Tokens per sec: 80778.11802699469, Loss: 2.2408480644226074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13189 , TFLOPS: 99.47683399551418, Tokens per sec: 81285.0818192534, Loss: 2.2539119720458984 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13190 , TFLOPS: 99.46315665564192, Tokens per sec: 81273.90571275773, Loss: 2.2543060779571533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13191 , TFLOPS: 99.60007619369371, Tokens per sec: 81385.78619192232, Loss: 2.2518529891967773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13192 , TFLOPS: 98.40659979166954, Tokens per sec: 80410.5659010126, Loss: 2.244253158569336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13193 , TFLOPS: 100.58053624401924, Tokens per sec: 82186.94533833036, Loss: 2.2645726203918457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13194 , TFLOPS: 99.47052342812893, Tokens per sec: 81279.92529219447, Loss: 2.2436537742614746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13195 , TFLOPS: 99.83530541127674, Tokens per sec: 81577.99804094816, Loss: 2.2454380989074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13196 , TFLOPS: 100.08095343586592, Tokens per sec: 81778.72336538271, Loss: 2.2548434734344482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13197 , TFLOPS: 100.70434447391204, Tokens per sec: 82288.11223008313, Loss: 2.2458486557006836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13198 , TFLOPS: 100.24889452048515, Tokens per sec: 81915.9523488132, Loss: 2.273383378982544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13199 , TFLOPS: 100.60195688566012, Tokens per sec: 82204.44869603148, Loss: 2.2708637714385986 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13200 , TFLOPS: 100.05589429790327, Tokens per sec: 81758.24689867382, Loss: 2.2351810932159424 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/model.pt +[rank0]:[2024-08-29 14:37:02,252] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007670291990507394, 'preprocessing_with_comm': 0.0016806419880595058, 'state_converting': 2.4220381409977563, : 2.4330423589999555}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013200, took 14.84s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011200 + eval ppl=7.8822855949401855, eval loss=2.064617872238159 +------------------------------------------------------------------ +iteration: 13201 , TFLOPS: 98.5798011347509, Tokens per sec: 80552.0931770436, Loss: 2.2718634605407715 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13202 , TFLOPS: 97.51534598937585, Tokens per sec: 79682.2994762445, Loss: 2.2500534057617188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13203 , TFLOPS: 99.5386506176079, Tokens per sec: 81335.5937724676, Loss: 2.2648444175720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13204 , TFLOPS: 98.87687893310874, Tokens per sec: 80794.84309354467, Loss: 2.2293429374694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13205 , TFLOPS: 99.665941628514, Tokens per sec: 81439.60653424112, Loss: 2.2562479972839355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13206 , TFLOPS: 99.5616388540814, Tokens per sec: 81354.37805225996, Loss: 2.2434284687042236 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13207 , TFLOPS: 100.6840319625835, Tokens per sec: 82271.51435418603, Loss: 2.253521203994751 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13208 , TFLOPS: 99.7057907407286, Tokens per sec: 81472.16826963906, Loss: 2.257033109664917 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13209 , TFLOPS: 100.55578391239003, Tokens per sec: 82166.71957097443, Loss: 2.2550835609436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13210 , TFLOPS: 100.59587867280767, Tokens per sec: 82199.48203184265, Loss: 2.2551674842834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13211 , TFLOPS: 100.7174575511019, Tokens per sec: 82298.8272630156, Loss: 2.239285469055176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13212 , TFLOPS: 100.56766106069935, Tokens per sec: 82176.42469460297, Loss: 2.2476706504821777 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13213 , TFLOPS: 101.32463641287653, Tokens per sec: 82794.96874114606, Loss: 2.236985206604004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13214 , TFLOPS: 99.2732378705082, Tokens per sec: 81118.71818448052, Loss: 2.2532577514648438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13215 , TFLOPS: 101.35345973374129, Tokens per sec: 82818.52101860294, Loss: 2.2672157287597656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13216 , TFLOPS: 99.33316498558035, Tokens per sec: 81167.68617286715, Loss: 2.244123935699463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13217 , TFLOPS: 99.92957344622987, Tokens per sec: 81655.02687898348, Loss: 2.2441205978393555 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13218 , TFLOPS: 99.92312483472845, Tokens per sec: 81649.75755252365, Loss: 2.2651102542877197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13219 , TFLOPS: 101.21400512523005, Tokens per sec: 82704.56906810742, Loss: 2.27063250541687 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13220 , TFLOPS: 100.06241226481016, Tokens per sec: 81763.57289722085, Loss: 2.2520270347595215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13221 , TFLOPS: 100.11872113790308, Tokens per sec: 81809.58432694442, Loss: 2.2425410747528076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13222 , TFLOPS: 100.03524478515942, Tokens per sec: 81741.37365024508, Loss: 2.243046998977661 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13223 , TFLOPS: 100.67322759898047, Tokens per sec: 82262.68583055693, Loss: 2.255276918411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13224 , TFLOPS: 98.16804336529012, Tokens per sec: 80215.63530402925, Loss: 2.245940923690796 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13225 , TFLOPS: 100.58948302490175, Tokens per sec: 82194.25598330009, Loss: 2.2457823753356934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13226 , TFLOPS: 99.7559948636584, Tokens per sec: 81513.1913508541, Loss: 2.2729995250701904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13227 , TFLOPS: 99.39231934476526, Tokens per sec: 81216.02272252581, Loss: 2.2446889877319336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13228 , TFLOPS: 100.04628156204147, Tokens per sec: 81750.39208474732, Loss: 2.24247670173645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13229 , TFLOPS: 99.46873585908142, Tokens per sec: 81278.46462350962, Loss: 2.256401300430298 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13230 , TFLOPS: 98.93200633611944, Tokens per sec: 80839.88911365031, Loss: 2.2625298500061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13231 , TFLOPS: 99.36396121099891, Tokens per sec: 81192.85056142202, Loss: 2.2517988681793213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13232 , TFLOPS: 100.59029066819517, Tokens per sec: 82194.91592942573, Loss: 2.2567617893218994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13233 , TFLOPS: 96.67442905759536, Tokens per sec: 78995.16460415906, Loss: 2.2585368156433105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13234 , TFLOPS: 100.55352654969609, Tokens per sec: 82164.87502180751, Loss: 2.2628653049468994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13235 , TFLOPS: 98.1514399976991, Tokens per sec: 80202.06826496193, Loss: 2.2446539402008057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13236 , TFLOPS: 98.20265947521031, Tokens per sec: 80243.92101854298, Loss: 2.2827625274658203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13237 , TFLOPS: 98.68230382145666, Tokens per sec: 80635.85076100478, Loss: 2.2631101608276367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13238 , TFLOPS: 99.13388996220438, Tokens per sec: 81004.85342147117, Loss: 2.242267370223999 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13239 , TFLOPS: 98.95641637702354, Tokens per sec: 80859.8351864434, Loss: 2.246063709259033 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13240 , TFLOPS: 98.92669312625605, Tokens per sec: 80835.54755309675, Loss: 2.2502784729003906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13241 , TFLOPS: 99.02627199340101, Tokens per sec: 80916.91600882896, Loss: 2.250274896621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13242 , TFLOPS: 100.1715282426849, Tokens per sec: 81852.73437163745, Loss: 2.25124192237854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13243 , TFLOPS: 100.14552335444483, Tokens per sec: 81831.4851080311, Loss: 2.2731077671051025 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13244 , TFLOPS: 99.55000296085939, Tokens per sec: 81344.87006437371, Loss: 2.243806838989258 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13245 , TFLOPS: 100.79811512444262, Tokens per sec: 82364.73464250295, Loss: 2.252206563949585 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13246 , TFLOPS: 98.92733139890245, Tokens per sec: 80836.0691021067, Loss: 2.229290008544922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13247 , TFLOPS: 100.67379000315279, Tokens per sec: 82263.1453854837, Loss: 2.2601945400238037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13248 , TFLOPS: 100.58543326390146, Tokens per sec: 82190.94681933662, Loss: 2.2507803440093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13249 , TFLOPS: 100.05638833509113, Tokens per sec: 81758.65058917766, Loss: 2.2610578536987305 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13250 , TFLOPS: 100.05927124381368, Tokens per sec: 81761.00628810754, Loss: 2.2490103244781494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13251 , TFLOPS: 100.63069925135888, Tokens per sec: 82227.93482293826, Loss: 2.2444353103637695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13252 , TFLOPS: 100.6249476671487, Tokens per sec: 82223.23505542119, Loss: 2.2405357360839844 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13253 , TFLOPS: 101.32791229144082, Tokens per sec: 82797.64554584956, Loss: 2.2679030895233154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13254 , TFLOPS: 98.85692363150538, Tokens per sec: 80778.5371029094, Loss: 2.2621378898620605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13255 , TFLOPS: 99.89000197370646, Tokens per sec: 81622.69201012427, Loss: 2.268282175064087 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13256 , TFLOPS: 101.32352786520305, Tokens per sec: 82794.06291830551, Loss: 2.247112274169922 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13257 , TFLOPS: 100.58729747915335, Tokens per sec: 82192.47011760811, Loss: 2.257538080215454 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13258 , TFLOPS: 100.09896909967338, Tokens per sec: 81793.44442804423, Loss: 2.243917465209961 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13259 , TFLOPS: 99.23421239616944, Tokens per sec: 81086.82946479322, Loss: 2.263854503631592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13260 , TFLOPS: 99.9958889846915, Tokens per sec: 81709.2150125743, Loss: 2.2436842918395996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13261 , TFLOPS: 100.10852768095188, Tokens per sec: 81801.25499086666, Loss: 2.262883186340332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13262 , TFLOPS: 99.6004472190544, Tokens per sec: 81386.0893662956, Loss: 2.2730674743652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13263 , TFLOPS: 100.04429951475522, Tokens per sec: 81748.77250288728, Loss: 2.255366086959839 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13264 , TFLOPS: 99.45334957672895, Tokens per sec: 81265.89209612127, Loss: 2.2634425163269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13265 , TFLOPS: 99.5658162044299, Tokens per sec: 81357.79147276428, Loss: 2.2532691955566406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13266 , TFLOPS: 100.05755598686589, Tokens per sec: 81759.60470750085, Loss: 2.2813913822174072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13267 , TFLOPS: 100.15555317226132, Tokens per sec: 81839.68073035944, Loss: 2.2694013118743896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13268 , TFLOPS: 97.76246389249911, Tokens per sec: 79884.22587626727, Loss: 2.2790589332580566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13269 , TFLOPS: 100.12817844174731, Tokens per sec: 81817.31213336819, Loss: 2.257622241973877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13270 , TFLOPS: 100.19622465004244, Tokens per sec: 81872.91444183137, Loss: 2.2651052474975586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13271 , TFLOPS: 97.68064964421974, Tokens per sec: 79817.37334791209, Loss: 2.270355463027954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13272 , TFLOPS: 100.66020779581596, Tokens per sec: 82252.04701423166, Loss: 2.25223445892334 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13273 , TFLOPS: 98.39409036364927, Tokens per sec: 80400.34412535591, Loss: 2.2742507457733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13274 , TFLOPS: 98.53844723636188, Tokens per sec: 80518.30184212589, Loss: 2.247709274291992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13275 , TFLOPS: 99.0638831688379, Tokens per sec: 80947.64906847625, Loss: 2.267972946166992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13276 , TFLOPS: 98.854527973416, Tokens per sec: 80776.57954901483, Loss: 2.263875961303711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13277 , TFLOPS: 99.09534143377684, Tokens per sec: 80973.35442656564, Loss: 2.264350414276123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13278 , TFLOPS: 97.5651715579439, Tokens per sec: 79723.0132309457, Loss: 2.2332990169525146 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13279 , TFLOPS: 99.50818989635489, Tokens per sec: 81310.70353300305, Loss: 2.2587029933929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13280 , TFLOPS: 99.57067546116855, Tokens per sec: 81361.76209653383, Loss: 2.260570526123047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13281 , TFLOPS: 100.12528052204128, Tokens per sec: 81814.9441685773, Loss: 2.252674102783203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13282 , TFLOPS: 98.95949480153385, Tokens per sec: 80862.35064635648, Loss: 2.248195171356201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13283 , TFLOPS: 100.859723350769, Tokens per sec: 82415.07631017108, Loss: 2.241773843765259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13284 , TFLOPS: 99.43787589499215, Tokens per sec: 81253.24815244613, Loss: 2.2405025959014893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13285 , TFLOPS: 101.28610257408381, Tokens per sec: 82763.48174951928, Loss: 2.2453205585479736 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13286 , TFLOPS: 99.44906343565405, Tokens per sec: 81262.38977991388, Loss: 2.2594237327575684 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13287 , TFLOPS: 100.44191696738618, Tokens per sec: 82073.67595901585, Loss: 2.2532777786254883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13288 , TFLOPS: 100.57365148509766, Tokens per sec: 82181.31962458607, Loss: 2.257554769515991 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13289 , TFLOPS: 100.69564376946363, Tokens per sec: 82281.00266050229, Loss: 2.2689099311828613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13290 , TFLOPS: 100.69140779046808, Tokens per sec: 82277.54133302122, Loss: 2.252110481262207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13291 , TFLOPS: 100.73143781547391, Tokens per sec: 82310.250896918, Loss: 2.254354953765869 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13292 , TFLOPS: 99.33358724992829, Tokens per sec: 81168.0312159359, Loss: 2.251739025115967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13293 , TFLOPS: 99.42492920816079, Tokens per sec: 81242.66908135886, Loss: 2.256817102432251 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13294 , TFLOPS: 101.32808818789297, Tokens per sec: 82797.78927536859, Loss: 2.2472827434539795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13295 , TFLOPS: 99.55816725220576, Tokens per sec: 81351.54131700039, Loss: 2.240626811981201 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13296 , TFLOPS: 100.10559964875087, Tokens per sec: 81798.86242038096, Loss: 2.2335426807403564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13297 , TFLOPS: 100.09249372211775, Tokens per sec: 81788.15323035247, Loss: 2.2448177337646484 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13298 , TFLOPS: 99.95179197286201, Tokens per sec: 81673.18220905036, Loss: 2.2333285808563232 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13299 , TFLOPS: 100.12232407657547, Tokens per sec: 81812.52838088151, Loss: 2.2753565311431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13300 , TFLOPS: 98.98987551285387, Tokens per sec: 80887.17550764515, Loss: 2.238083600997925 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13301 , TFLOPS: 97.15915998637128, Tokens per sec: 79391.25072414604, Loss: 2.2642698287963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13302 , TFLOPS: 99.57374706417816, Tokens per sec: 81364.27198242307, Loss: 2.2707431316375732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13303 , TFLOPS: 100.15871747267538, Tokens per sec: 81842.26636168412, Loss: 2.25557279586792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13304 , TFLOPS: 100.10838449223829, Tokens per sec: 81801.13798768279, Loss: 2.2518491744995117 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13305 , TFLOPS: 100.70698202254225, Tokens per sec: 82290.26743896541, Loss: 2.2520482540130615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13306 , TFLOPS: 98.1913449440516, Tokens per sec: 80234.67562387126, Loss: 2.239137649536133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13307 , TFLOPS: 100.0399674668128, Tokens per sec: 81745.23267499669, Loss: 2.250933885574341 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13308 , TFLOPS: 100.68553975841618, Tokens per sec: 82272.74641297503, Loss: 2.2333083152770996 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13309 , TFLOPS: 97.53610912764498, Tokens per sec: 79699.26557101494, Loss: 2.2565250396728516 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13310 , TFLOPS: 100.11540620469954, Tokens per sec: 81806.87560968983, Loss: 2.251218795776367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13311 , TFLOPS: 98.6733882854003, Tokens per sec: 80628.56564698681, Loss: 2.2620410919189453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13312 , TFLOPS: 99.20449017171143, Tokens per sec: 81062.54267007046, Loss: 2.265738010406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13313 , TFLOPS: 98.21038085553803, Tokens per sec: 80250.23035717478, Loss: 2.253588914871216 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13314 , TFLOPS: 99.01213035208704, Tokens per sec: 80905.36050967309, Loss: 2.231891632080078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13315 , TFLOPS: 98.86035070598588, Tokens per sec: 80781.33745368816, Loss: 2.2592971324920654 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13316 , TFLOPS: 99.09174423135396, Tokens per sec: 80970.41505986566, Loss: 2.238179922103882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13317 , TFLOPS: 99.42366797561588, Tokens per sec: 81241.6384957794, Loss: 2.261800527572632 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13318 , TFLOPS: 99.03824468968844, Tokens per sec: 80926.69920717005, Loss: 2.258145332336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13319 , TFLOPS: 100.17538721816214, Tokens per sec: 81855.88763983877, Loss: 2.268385171890259 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13320 , TFLOPS: 99.39597999832816, Tokens per sec: 81219.01393678566, Loss: 2.283637285232544 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13321 , TFLOPS: 100.1210473986672, Tokens per sec: 81811.48517449806, Loss: 2.2555294036865234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13322 , TFLOPS: 99.45097201084357, Tokens per sec: 81263.94932583233, Loss: 2.2382516860961914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13323 , TFLOPS: 100.5930527636842, Tokens per sec: 82197.17290875247, Loss: 2.2358953952789307 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13324 , TFLOPS: 99.7170432028498, Tokens per sec: 81481.36294610244, Loss: 2.255760431289673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13325 , TFLOPS: 99.46220538735119, Tokens per sec: 81273.12840694956, Loss: 2.2343480587005615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13326 , TFLOPS: 100.67123717026384, Tokens per sec: 82261.05940001438, Loss: 2.2523982524871826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13327 , TFLOPS: 99.92668961684268, Tokens per sec: 81652.67042775484, Loss: 2.275508403778076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13328 , TFLOPS: 100.68851229687236, Tokens per sec: 82275.17535066752, Loss: 2.2430686950683594 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13329 , TFLOPS: 100.66903481581542, Tokens per sec: 82259.25979950093, Loss: 2.2821362018585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13330 , TFLOPS: 99.32700813798928, Tokens per sec: 81162.65525420912, Loss: 2.2705912590026855 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13331 , TFLOPS: 99.45577230778117, Tokens per sec: 81267.87177203069, Loss: 2.2483391761779785 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13332 , TFLOPS: 100.73308028305539, Tokens per sec: 82311.5929994597, Loss: 2.253331422805786 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13333 , TFLOPS: 101.19431033249505, Tokens per sec: 82688.47594597458, Loss: 2.2780601978302 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13334 , TFLOPS: 99.5449669071133, Tokens per sec: 81340.75497521822, Loss: 2.2595043182373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13335 , TFLOPS: 100.6889787126484, Tokens per sec: 82275.55647100473, Loss: 2.240830898284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13336 , TFLOPS: 99.2303849317088, Tokens per sec: 81083.70194505474, Loss: 2.2393784523010254 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13337 , TFLOPS: 100.51741664260041, Tokens per sec: 82135.36868716794, Loss: 2.2613282203674316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13338 , TFLOPS: 99.40884625032326, Tokens per sec: 81229.52728249766, Loss: 2.247255325317383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13339 , TFLOPS: 100.69430390538082, Tokens per sec: 82279.90782306905, Loss: 2.2341246604919434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13340 , TFLOPS: 100.12624186257317, Tokens per sec: 81815.72970467301, Loss: 2.268383502960205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13341 , TFLOPS: 100.04134846042403, Tokens per sec: 81746.36112042637, Loss: 2.2618978023529053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13342 , TFLOPS: 98.67959496423337, Tokens per sec: 80633.63728403545, Loss: 2.2319493293762207 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13343 , TFLOPS: 100.67769376528975, Tokens per sec: 82266.33525001758, Loss: 2.2482051849365234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13344 , TFLOPS: 98.89505548655922, Tokens per sec: 80809.69562327515, Loss: 2.2655532360076904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13345 , TFLOPS: 100.01381589697239, Tokens per sec: 81723.8635540788, Loss: 2.2459354400634766 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13346 , TFLOPS: 100.0595924654913, Tokens per sec: 81761.26876660931, Loss: 2.269632339477539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13347 , TFLOPS: 97.51371606521616, Tokens per sec: 79680.96762325564, Loss: 2.268735647201538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13348 , TFLOPS: 100.75873753962583, Tokens per sec: 82332.55820427995, Loss: 2.274962902069092 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13349 , TFLOPS: 98.91441504386609, Tokens per sec: 80825.51481590993, Loss: 2.2419521808624268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13350 , TFLOPS: 99.0172111663125, Tokens per sec: 80909.51217376853, Loss: 2.24802303314209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13351 , TFLOPS: 98.40004692249695, Tokens per sec: 80405.21138292583, Loss: 2.262648105621338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13352 , TFLOPS: 99.62475709585996, Tokens per sec: 81405.95359242531, Loss: 2.248037815093994 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13353 , TFLOPS: 98.98330750180011, Tokens per sec: 80881.80861673747, Loss: 2.2414286136627197 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13354 , TFLOPS: 98.464140379486, Tokens per sec: 80457.58379654394, Loss: 2.251495838165283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13355 , TFLOPS: 98.99785600961924, Tokens per sec: 80893.6965769884, Loss: 2.2417054176330566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13356 , TFLOPS: 100.03838455961618, Tokens per sec: 81743.93924077868, Loss: 2.2607779502868652 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13357 , TFLOPS: 98.9224225171275, Tokens per sec: 80832.05792844255, Loss: 2.248835563659668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13358 , TFLOPS: 100.3140956848685, Tokens per sec: 81969.22989865813, Loss: 2.2439074516296387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13359 , TFLOPS: 99.46661837486442, Tokens per sec: 81276.73437264701, Loss: 2.276047468185425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13360 , TFLOPS: 100.13393213870579, Tokens per sec: 81822.01362726631, Loss: 2.2506730556488037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13361 , TFLOPS: 99.96676983354143, Tokens per sec: 81685.42100457595, Loss: 2.2591419219970703 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13362 , TFLOPS: 99.99333000042697, Tokens per sec: 81707.12400065764, Loss: 2.2477381229400635 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13363 , TFLOPS: 100.02729968543603, Tokens per sec: 81734.88150473603, Loss: 2.243448257446289 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13364 , TFLOPS: 100.09508334326841, Tokens per sec: 81790.26927645746, Loss: 2.2506446838378906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13365 , TFLOPS: 100.67110396222918, Tokens per sec: 82260.95055230017, Loss: 2.2482516765594482 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13366 , TFLOPS: 98.98001841695904, Tokens per sec: 80879.12102084525, Loss: 2.2527849674224854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13367 , TFLOPS: 100.7364974703733, Tokens per sec: 82314.38527118339, Loss: 2.2414112091064453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13368 , TFLOPS: 99.98109758044481, Tokens per sec: 81697.1285753997, Loss: 2.232146978378296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13369 , TFLOPS: 99.28176547064878, Tokens per sec: 81125.68630607532, Loss: 2.2645297050476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13370 , TFLOPS: 100.52185107160412, Tokens per sec: 82138.99217325923, Loss: 2.257416009902954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13371 , TFLOPS: 101.38448735274191, Tokens per sec: 82843.87448481032, Loss: 2.239936590194702 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13372 , TFLOPS: 100.06135686129684, Tokens per sec: 81762.71049984162, Loss: 2.254753351211548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13373 , TFLOPS: 100.16587571612448, Tokens per sec: 81848.11554667594, Loss: 2.250298500061035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13374 , TFLOPS: 99.20722044551665, Tokens per sec: 81064.77364708022, Loss: 2.2461166381835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13375 , TFLOPS: 100.11780772415655, Tokens per sec: 81808.83795305876, Loss: 2.267317533493042 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13376 , TFLOPS: 100.0013989968426, Tokens per sec: 81713.7173853433, Loss: 2.2408390045166016 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13377 , TFLOPS: 98.93823635715374, Tokens per sec: 80844.97982421244, Loss: 2.247464895248413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13378 , TFLOPS: 99.97401881961308, Tokens per sec: 81691.34433770059, Loss: 2.265292167663574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13379 , TFLOPS: 99.43226988195501, Tokens per sec: 81248.66733488187, Loss: 2.277947187423706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13380 , TFLOPS: 99.37610174858625, Tokens per sec: 81202.77090720987, Loss: 2.270157814025879 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13381 , TFLOPS: 100.10433821376351, Tokens per sec: 81797.83166938057, Loss: 2.2627100944519043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13382 , TFLOPS: 99.37905011163154, Tokens per sec: 81205.1800905518, Loss: 2.245054006576538 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13383 , TFLOPS: 99.1379203093635, Tokens per sec: 81008.14672188525, Loss: 2.26389741897583 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13384 , TFLOPS: 98.98523650931648, Tokens per sec: 80883.38485843615, Loss: 2.24149751663208 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13385 , TFLOPS: 98.84765900474898, Tokens per sec: 80770.96673789404, Loss: 2.2900187969207764 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13386 , TFLOPS: 99.54949917267356, Tokens per sec: 81344.45840607841, Loss: 2.255457639694214 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13387 , TFLOPS: 98.71218595752917, Tokens per sec: 80660.26822362424, Loss: 2.2639565467834473 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13388 , TFLOPS: 100.10583806518918, Tokens per sec: 81799.05723659007, Loss: 2.258960247039795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13389 , TFLOPS: 98.47568698021604, Tokens per sec: 80467.0188212364, Loss: 2.275618076324463 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13390 , TFLOPS: 100.14910505965571, Tokens per sec: 81834.41181156022, Loss: 2.2532427310943604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13391 , TFLOPS: 99.5082851145898, Tokens per sec: 81310.78133827393, Loss: 2.235853433609009 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13392 , TFLOPS: 99.19294059526935, Tokens per sec: 81053.1052138471, Loss: 2.2428274154663086 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13393 , TFLOPS: 98.77819786197824, Tokens per sec: 80714.20825004716, Loss: 2.2467784881591797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13394 , TFLOPS: 99.53444264051457, Tokens per sec: 81332.15532606168, Loss: 2.2493669986724854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13395 , TFLOPS: 100.74621665514267, Tokens per sec: 82322.32706725253, Loss: 2.2487905025482178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13396 , TFLOPS: 99.50543638397373, Tokens per sec: 81308.45356715469, Loss: 2.256213426589966 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13397 , TFLOPS: 101.22866309351615, Tokens per sec: 82716.54647132366, Loss: 2.244988441467285 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13398 , TFLOPS: 99.4839280220526, Tokens per sec: 81290.87853095445, Loss: 2.2702980041503906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13399 , TFLOPS: 101.28929747641979, Tokens per sec: 82766.0923864621, Loss: 2.252141237258911 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13400 , TFLOPS: 99.885838612129, Tokens per sec: 81619.29002020488, Loss: 2.2509067058563232 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/model.pt +[rank0]:[2024-08-29 15:31:02,742] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.007691157006775029, 'preprocessing_with_comm': 0.0017939109966391698, 'state_converting': 2.4034948290063767, : 2.4146248329925584}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013400, took 14.54s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011400 + eval ppl=7.464942455291748, eval loss=2.0102176666259766 +------------------------------------------------------------------ +iteration: 13401 , TFLOPS: 98.04441838744785, Tokens per sec: 80114.61815224428, Loss: 2.2572994232177734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13402 , TFLOPS: 98.29133454052446, Tokens per sec: 80316.37969711043, Loss: 2.2558693885803223 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13403 , TFLOPS: 99.73183652426718, Tokens per sec: 81493.45094984656, Loss: 2.265115737915039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13404 , TFLOPS: 100.1877910898306, Tokens per sec: 81866.02316268312, Loss: 2.246814250946045 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13405 , TFLOPS: 98.41869535395851, Tokens per sec: 80420.4494963264, Loss: 2.2638986110687256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13406 , TFLOPS: 100.84846211199346, Tokens per sec: 82405.8744620775, Loss: 2.23041033744812 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13407 , TFLOPS: 100.8647956734318, Tokens per sec: 82419.22103559205, Loss: 2.249591112136841 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13408 , TFLOPS: 100.16770982665695, Tokens per sec: 81849.61424560624, Loss: 2.2650716304779053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13409 , TFLOPS: 101.26462395451524, Tokens per sec: 82745.93101656093, Loss: 2.2347381114959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13410 , TFLOPS: 100.53107177060187, Tokens per sec: 82146.5266437722, Loss: 2.2805473804473877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13411 , TFLOPS: 100.61630471093407, Tokens per sec: 82216.17267340871, Loss: 2.256810188293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13412 , TFLOPS: 101.25463728022439, Tokens per sec: 82737.77064791802, Loss: 2.272953510284424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13413 , TFLOPS: 100.00073683680279, Tokens per sec: 81713.17631732924, Loss: 2.2480721473693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13414 , TFLOPS: 100.38526188405726, Tokens per sec: 82027.3816319962, Loss: 2.2485105991363525 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13415 , TFLOPS: 101.20944819436264, Tokens per sec: 82700.84548258981, Loss: 2.2508513927459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13416 , TFLOPS: 99.88733468829832, Tokens per sec: 81620.51250255526, Loss: 2.2457644939422607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13417 , TFLOPS: 100.04691887842314, Tokens per sec: 81750.91285236877, Loss: 2.2677295207977295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13418 , TFLOPS: 101.25360171426382, Tokens per sec: 82736.92446031379, Loss: 2.25205397605896 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13419 , TFLOPS: 99.9932159668545, Tokens per sec: 81707.03082089011, Loss: 2.271982431411743 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13420 , TFLOPS: 100.6240612971071, Tokens per sec: 82222.51077964301, Loss: 2.2451391220092773 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13421 , TFLOPS: 99.96952864622514, Tokens per sec: 81687.67530143796, Loss: 2.2352309226989746 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13422 , TFLOPS: 99.82156096371926, Tokens per sec: 81566.76709903541, Loss: 2.2621231079101562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13423 , TFLOPS: 100.16507073712033, Tokens per sec: 81847.45777760966, Loss: 2.24880051612854 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13424 , TFLOPS: 100.08898941879441, Tokens per sec: 81785.28977389818, Loss: 2.2313265800476074 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13425 , TFLOPS: 100.65792743613814, Tokens per sec: 82250.18367363722, Loss: 2.2753987312316895 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13426 , TFLOPS: 100.6181387584224, Tokens per sec: 82217.67132082403, Loss: 2.2380921840667725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13427 , TFLOPS: 98.36183855397105, Tokens per sec: 80373.9903414324, Loss: 2.2631113529205322 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13428 , TFLOPS: 100.61627444642981, Tokens per sec: 82216.14794350322, Loss: 2.2487897872924805 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13429 , TFLOPS: 101.32860660882707, Tokens per sec: 82798.21289045918, Loss: 2.2744646072387695 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13430 , TFLOPS: 97.38094056037264, Tokens per sec: 79572.47334030243, Loss: 2.2524452209472656 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13431 , TFLOPS: 100.54018402051364, Tokens per sec: 82153.97249774529, Loss: 2.254429578781128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13432 , TFLOPS: 99.48746942602796, Tokens per sec: 81293.77230330656, Loss: 2.2658467292785645 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13433 , TFLOPS: 98.62350360059098, Tokens per sec: 80587.80358688309, Loss: 2.2710683345794678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13434 , TFLOPS: 99.54183280675284, Tokens per sec: 81338.19402113477, Loss: 2.25586199760437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13435 , TFLOPS: 99.320397747538, Tokens per sec: 81157.2537340048, Loss: 2.2509424686431885 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13436 , TFLOPS: 98.91570165801456, Tokens per sec: 80826.56614145105, Loss: 2.2556235790252686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13437 , TFLOPS: 98.91398828010783, Tokens per sec: 80825.16609625719, Loss: 2.260866165161133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13438 , TFLOPS: 99.44857369606794, Tokens per sec: 81261.98960109106, Loss: 2.262131452560425 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13439 , TFLOPS: 98.7137843427671, Tokens per sec: 80661.57430534824, Loss: 2.2675654888153076 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13440 , TFLOPS: 98.46941945353262, Tokens per sec: 80461.89746384289, Loss: 2.2538609504699707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13441 , TFLOPS: 99.1381803007495, Tokens per sec: 81008.35916753953, Loss: 2.255293846130371 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13442 , TFLOPS: 99.57322136126058, Tokens per sec: 81363.84241703633, Loss: 2.2537429332733154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13443 , TFLOPS: 98.34071250098904, Tokens per sec: 80356.72769970783, Loss: 2.2626092433929443 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13444 , TFLOPS: 100.89252935803039, Tokens per sec: 82441.88294320612, Loss: 2.2692673206329346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13445 , TFLOPS: 100.84695801525908, Tokens per sec: 82404.64542591695, Loss: 2.2643070220947266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13446 , TFLOPS: 100.14542000443225, Tokens per sec: 81831.40065807538, Loss: 2.2649402618408203 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13447 , TFLOPS: 101.35741572595543, Tokens per sec: 82821.75356167743, Loss: 2.2582452297210693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13448 , TFLOPS: 101.34770365155707, Tokens per sec: 82813.81757567546, Loss: 2.2474477291107178 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13449 , TFLOPS: 100.75682146593942, Tokens per sec: 82330.99253114665, Loss: 2.2446224689483643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13450 , TFLOPS: 101.27349879619356, Tokens per sec: 82753.18287815504, Loss: 2.2571072578430176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13451 , TFLOPS: 99.37314594643671, Tokens per sec: 81200.35564518436, Loss: 2.250457286834717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13452 , TFLOPS: 100.57115800408475, Tokens per sec: 82179.28213706252, Loss: 2.25469970703125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13453 , TFLOPS: 101.31079604505938, Tokens per sec: 82783.6594203199, Loss: 2.2544968128204346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13454 , TFLOPS: 100.72036263889996, Tokens per sec: 82301.201085039, Loss: 2.2602109909057617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13455 , TFLOPS: 98.79255351115778, Tokens per sec: 80725.93862053906, Loss: 2.2553439140319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13456 , TFLOPS: 101.29926768131313, Tokens per sec: 82774.23929753635, Loss: 2.2657995223999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13457 , TFLOPS: 101.24002834727546, Tokens per sec: 82725.83331273805, Loss: 2.256679058074951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13458 , TFLOPS: 99.95410243737965, Tokens per sec: 81675.07014908436, Loss: 2.2547855377197266 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13459 , TFLOPS: 100.19481731887495, Tokens per sec: 81871.76447530648, Loss: 2.2695581912994385 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13460 , TFLOPS: 100.04853058944349, Tokens per sec: 81752.22982293263, Loss: 2.227679491043091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13461 , TFLOPS: 100.02653812792113, Tokens per sec: 81734.25921648637, Loss: 2.268206834793091 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13462 , TFLOPS: 99.32687797214295, Tokens per sec: 81162.54889234532, Loss: 2.2611498832702637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13463 , TFLOPS: 101.34476760376383, Tokens per sec: 82811.41845544302, Loss: 2.26600980758667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13464 , TFLOPS: 99.48542821454177, Tokens per sec: 81292.10437685573, Loss: 2.256415605545044 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13465 , TFLOPS: 99.48180348064554, Tokens per sec: 81289.14251348017, Loss: 2.2673983573913574 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13466 , TFLOPS: 99.57635195454255, Tokens per sec: 81366.40050539566, Loss: 2.260499954223633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13467 , TFLOPS: 101.37115444061784, Tokens per sec: 82832.97981909405, Loss: 2.2553651332855225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13468 , TFLOPS: 97.67331642254453, Tokens per sec: 79811.38118370734, Loss: 2.251129627227783 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13469 , TFLOPS: 100.64336038450449, Tokens per sec: 82238.28056075811, Loss: 2.266693353652954 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13470 , TFLOPS: 99.52069944726689, Tokens per sec: 81320.92540907772, Loss: 2.2495856285095215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13471 , TFLOPS: 98.44080173392294, Tokens per sec: 80438.51318846419, Loss: 2.249324321746826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13472 , TFLOPS: 99.29960900358657, Tokens per sec: 81140.2667161722, Loss: 2.2610392570495605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13473 , TFLOPS: 98.89882199527565, Tokens per sec: 80812.77333450592, Loss: 2.255697250366211 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13474 , TFLOPS: 98.9270227330682, Tokens per sec: 80835.8168833078, Loss: 2.2421045303344727 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13475 , TFLOPS: 98.81140173517963, Tokens per sec: 80741.33998958376, Loss: 2.250335693359375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13476 , TFLOPS: 100.65218070816687, Tokens per sec: 82245.48787427797, Loss: 2.2521913051605225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13477 , TFLOPS: 99.06654610077626, Tokens per sec: 80949.82501871305, Loss: 2.2581517696380615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13478 , TFLOPS: 99.59871749453589, Tokens per sec: 81384.67596386444, Loss: 2.2467312812805176 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13479 , TFLOPS: 99.94665149123755, Tokens per sec: 81668.98178918725, Loss: 2.2461109161376953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13480 , TFLOPS: 99.39402413204724, Tokens per sec: 81217.41574809904, Loss: 2.2592763900756836 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13481 , TFLOPS: 99.5957848471166, Tokens per sec: 81382.27962216508, Loss: 2.261770725250244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13482 , TFLOPS: 99.43631665767943, Tokens per sec: 81251.97405949957, Loss: 2.2453503608703613 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13483 , TFLOPS: 100.0612890726783, Tokens per sec: 81762.65510801636, Loss: 2.2513511180877686 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13484 , TFLOPS: 99.67655966085205, Tokens per sec: 81448.282801797, Loss: 2.22225284576416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13485 , TFLOPS: 101.21498471049034, Tokens per sec: 82705.36951244037, Loss: 2.264946460723877 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13486 , TFLOPS: 101.26412326571445, Tokens per sec: 82745.52189085292, Loss: 2.2539591789245605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13487 , TFLOPS: 99.46795507266309, Tokens per sec: 81277.82662282798, Loss: 2.246124744415283 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13488 , TFLOPS: 101.29001863874912, Tokens per sec: 82766.68166676577, Loss: 2.2478063106536865 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13489 , TFLOPS: 101.3097562872889, Tokens per sec: 82782.8098074799, Loss: 2.2371201515197754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13490 , TFLOPS: 99.48967655721316, Tokens per sec: 81295.5758070146, Loss: 2.238654375076294 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13491 , TFLOPS: 101.2829997140401, Tokens per sec: 82760.94632270283, Loss: 2.2598040103912354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13492 , TFLOPS: 100.67732025131706, Tokens per sec: 82266.0300421354, Loss: 2.2297585010528564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13493 , TFLOPS: 98.09879162812484, Tokens per sec: 80159.04792689318, Loss: 2.2666819095611572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13494 , TFLOPS: 101.29188323412193, Tokens per sec: 82768.20527564384, Loss: 2.277621030807495 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13495 , TFLOPS: 101.33452863848115, Tokens per sec: 82803.05193333614, Loss: 2.263787031173706 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13496 , TFLOPS: 99.43027783532547, Tokens per sec: 81247.03958230103, Loss: 2.242698907852173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13497 , TFLOPS: 100.71187887411726, Tokens per sec: 82294.26878244354, Loss: 2.2447385787963867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13498 , TFLOPS: 98.5969553050767, Tokens per sec: 80566.11029120446, Loss: 2.2591235637664795 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13499 , TFLOPS: 100.50526482543923, Tokens per sec: 82125.43912454993, Loss: 2.264005661010742 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13500 , TFLOPS: 98.82057491639364, Tokens per sec: 80748.83563209242, Loss: 2.2909188270568848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13501 , TFLOPS: 101.23435817982195, Tokens per sec: 82721.2000729487, Loss: 2.280546188354492 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13502 , TFLOPS: 99.27058362122692, Tokens per sec: 81116.54932906633, Loss: 2.258204221725464 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13503 , TFLOPS: 97.97503379330041, Tokens per sec: 80057.92221424806, Loss: 2.2450661659240723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13504 , TFLOPS: 100.55498874411907, Tokens per sec: 82166.06981851078, Loss: 2.25299072265625 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13505 , TFLOPS: 100.11989704368966, Tokens per sec: 81810.54518983312, Loss: 2.2635092735290527 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13506 , TFLOPS: 97.99479523710426, Tokens per sec: 80074.06979868501, Loss: 2.266627311706543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13507 , TFLOPS: 101.36654550832357, Tokens per sec: 82829.21374187194, Loss: 2.247981548309326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13508 , TFLOPS: 100.0942576247707, Tokens per sec: 81789.5945606171, Loss: 2.2579102516174316 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13509 , TFLOPS: 99.3390854017877, Tokens per sec: 81172.52389735523, Loss: 2.284276008605957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13510 , TFLOPS: 100.64815627547428, Tokens per sec: 82242.19940672674, Loss: 2.2633912563323975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13511 , TFLOPS: 99.35800914714294, Tokens per sec: 81187.98697682556, Loss: 2.276325225830078 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13512 , TFLOPS: 100.13650055409182, Tokens per sec: 81824.11234559513, Loss: 2.2635600566864014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13513 , TFLOPS: 99.55650024410534, Tokens per sec: 81350.17916176893, Loss: 2.24900221824646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13514 , TFLOPS: 98.9487600118941, Tokens per sec: 80853.57897340375, Loss: 2.2623159885406494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13515 , TFLOPS: 99.57354846064204, Tokens per sec: 81364.10969836121, Loss: 2.2426273822784424 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13516 , TFLOPS: 98.68459869160418, Tokens per sec: 80637.72595847755, Loss: 2.2467706203460693 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13517 , TFLOPS: 100.63220397431436, Tokens per sec: 82229.16437080021, Loss: 2.280778169631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13518 , TFLOPS: 99.37924485226344, Tokens per sec: 81205.3392181352, Loss: 2.239631414413452 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13519 , TFLOPS: 98.40181341933864, Tokens per sec: 80406.65483296884, Loss: 2.2477469444274902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13520 , TFLOPS: 99.96199441554708, Tokens per sec: 81681.51888760258, Loss: 2.233506441116333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13521 , TFLOPS: 100.7394926057904, Tokens per sec: 82316.83267343436, Loss: 2.2911343574523926 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13522 , TFLOPS: 98.52751638983501, Tokens per sec: 80509.36996604376, Loss: 2.2642784118652344 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13523 , TFLOPS: 101.40897428998808, Tokens per sec: 82863.88339158401, Loss: 2.229112148284912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13524 , TFLOPS: 100.61848092445108, Tokens per sec: 82217.95091349419, Loss: 2.255033254623413 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13525 , TFLOPS: 98.88113341352691, Tokens per sec: 80798.31953901442, Loss: 2.2554566860198975 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13526 , TFLOPS: 101.33267257066689, Tokens per sec: 82801.53529254567, Loss: 2.26717472076416 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13527 , TFLOPS: 99.86839772935365, Tokens per sec: 81605.03862592086, Loss: 2.2875332832336426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13528 , TFLOPS: 99.88687742212036, Tokens per sec: 81620.13885859017, Loss: 2.248182535171509 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13529 , TFLOPS: 101.32534360818939, Tokens per sec: 82795.54660864102, Loss: 2.268990993499756 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13530 , TFLOPS: 99.62785019259057, Tokens per sec: 81408.48104138728, Loss: 2.2584612369537354 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13531 , TFLOPS: 100.0437046884101, Tokens per sec: 81748.2864549685, Loss: 2.2419841289520264 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13532 , TFLOPS: 99.44924028576906, Tokens per sec: 81262.53428869539, Loss: 2.25276255607605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13533 , TFLOPS: 100.66957710802464, Tokens per sec: 82259.70292042493, Loss: 2.250380277633667 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13534 , TFLOPS: 99.16342936817601, Tokens per sec: 81028.99083050256, Loss: 2.263993263244629 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13535 , TFLOPS: 99.12962589555174, Tokens per sec: 81001.36914284246, Loss: 2.240328311920166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13536 , TFLOPS: 99.8717839595117, Tokens per sec: 81607.80560175232, Loss: 2.271610736846924 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13537 , TFLOPS: 100.16170578461647, Tokens per sec: 81844.70818829717, Loss: 2.2499117851257324 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13538 , TFLOPS: 98.3856820650646, Tokens per sec: 80393.47348813353, Loss: 2.242384433746338 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13539 , TFLOPS: 100.35122146569483, Tokens per sec: 81999.56632987394, Loss: 2.270329475402832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13540 , TFLOPS: 100.62247822117294, Tokens per sec: 82221.21720754518, Loss: 2.2756993770599365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13541 , TFLOPS: 98.88416958728618, Tokens per sec: 80800.80047475119, Loss: 2.273768901824951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13542 , TFLOPS: 98.47236128266718, Tokens per sec: 80464.30130815816, Loss: 2.2650563716888428 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13543 , TFLOPS: 100.71336988521666, Tokens per sec: 82295.4871259949, Loss: 2.258112907409668 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13544 , TFLOPS: 98.91674663359088, Tokens per sec: 80827.42001789453, Loss: 2.2758266925811768 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13545 , TFLOPS: 100.66702956949072, Tokens per sec: 82257.62126110938, Loss: 2.2683823108673096 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13546 , TFLOPS: 100.00779535636413, Tokens per sec: 81718.9440153653, Loss: 2.254363775253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13547 , TFLOPS: 97.62080809056796, Tokens per sec: 79768.47527396458, Loss: 2.245311737060547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13548 , TFLOPS: 99.23115907993028, Tokens per sec: 81084.33452149486, Loss: 2.2717819213867188 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13549 , TFLOPS: 100.03016116939015, Tokens per sec: 81737.21969693611, Loss: 2.2501797676086426 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13550 , TFLOPS: 99.58181537671814, Tokens per sec: 81370.86480829633, Loss: 2.263718605041504 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13551 , TFLOPS: 98.29531819295272, Tokens per sec: 80319.6348420579, Loss: 2.2549870014190674 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13552 , TFLOPS: 99.34941406579289, Tokens per sec: 81180.96371459734, Loss: 2.261540412902832 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13553 , TFLOPS: 98.88913932990255, Tokens per sec: 80804.86137937556, Loss: 2.2469491958618164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13554 , TFLOPS: 99.74428186756161, Tokens per sec: 81503.62036022407, Loss: 2.23647141456604 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13555 , TFLOPS: 100.00608718953266, Tokens per sec: 81717.54822827516, Loss: 2.2534728050231934 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13556 , TFLOPS: 100.09129393877951, Tokens per sec: 81787.17285650162, Loss: 2.246555805206299 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13557 , TFLOPS: 97.79268066762899, Tokens per sec: 79908.91678107488, Loss: 2.2685632705688477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13558 , TFLOPS: 100.23623096323784, Tokens per sec: 81905.60463019746, Loss: 2.2773690223693848 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13559 , TFLOPS: 100.05146159963984, Tokens per sec: 81754.62482681496, Loss: 2.2604949474334717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13560 , TFLOPS: 98.32732238543612, Tokens per sec: 80345.7862915973, Loss: 2.264233350753784 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13561 , TFLOPS: 101.34957085123771, Tokens per sec: 82815.34331260053, Loss: 2.238013744354248 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13562 , TFLOPS: 101.10191214688402, Tokens per sec: 82612.97500997092, Loss: 2.26336407661438 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13563 , TFLOPS: 98.75853454666829, Tokens per sec: 80698.14084892864, Loss: 2.2650792598724365 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13564 , TFLOPS: 101.2407801771768, Tokens per sec: 82726.44765230427, Loss: 2.240861415863037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13565 , TFLOPS: 100.04060368677371, Tokens per sec: 81745.7525467045, Loss: 2.2408885955810547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13566 , TFLOPS: 100.6349300985686, Tokens per sec: 82231.39195710504, Loss: 2.2708663940429688 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13567 , TFLOPS: 100.73912193872539, Tokens per sec: 82316.52979183372, Loss: 2.2485930919647217 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13568 , TFLOPS: 99.49102865193368, Tokens per sec: 81296.68063841674, Loss: 2.2410004138946533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13569 , TFLOPS: 99.25398872820286, Tokens per sec: 81102.9892147858, Loss: 2.2604711055755615 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13570 , TFLOPS: 98.83978601797024, Tokens per sec: 80764.53351772841, Loss: 2.2439188957214355 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13571 , TFLOPS: 100.69335046269772, Tokens per sec: 82279.1287405089, Loss: 2.245051860809326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13572 , TFLOPS: 100.0068972518914, Tokens per sec: 81718.21015108135, Loss: 2.261522054672241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13573 , TFLOPS: 99.04027010266726, Tokens per sec: 80928.35422425382, Loss: 2.2654008865356445 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13574 , TFLOPS: 99.93338817018048, Tokens per sec: 81658.14398812325, Loss: 2.2533411979675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13575 , TFLOPS: 98.55784689112795, Tokens per sec: 80534.15379942679, Loss: 2.26101016998291 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13576 , TFLOPS: 99.03609897094826, Tokens per sec: 80924.94588515173, Loss: 2.2365832328796387 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13577 , TFLOPS: 100.77011962991487, Tokens per sec: 82341.85880325618, Loss: 2.242443799972534 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13578 , TFLOPS: 100.11582400286774, Tokens per sec: 81807.21700332816, Loss: 2.268129348754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13579 , TFLOPS: 99.20162781994556, Tokens per sec: 81060.20376875906, Loss: 2.237870693206787 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13580 , TFLOPS: 99.99027822464086, Tokens per sec: 81704.63031610312, Loss: 2.2686314582824707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13581 , TFLOPS: 100.53974107940695, Tokens per sec: 82153.6105591647, Loss: 2.247044563293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13582 , TFLOPS: 98.9878821610882, Tokens per sec: 80885.54668860398, Loss: 2.2576210498809814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13583 , TFLOPS: 100.1565858607207, Tokens per sec: 81840.52456668337, Loss: 2.2436373233795166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13584 , TFLOPS: 99.3915214536351, Tokens per sec: 81215.37074514381, Loss: 2.254096031188965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13585 , TFLOPS: 98.24574377922059, Tokens per sec: 80279.12631243843, Loss: 2.277647018432617 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13586 , TFLOPS: 100.08562871510273, Tokens per sec: 81782.54365639949, Loss: 2.260112762451172 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13587 , TFLOPS: 99.37397871021412, Tokens per sec: 81201.03611790415, Loss: 2.252701997756958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13588 , TFLOPS: 99.45909184306917, Tokens per sec: 81270.58424976695, Loss: 2.244292974472046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13589 , TFLOPS: 99.0817267489956, Tokens per sec: 80962.22951715773, Loss: 2.269075393676758 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13590 , TFLOPS: 99.39210844533635, Tokens per sec: 81215.85039117342, Loss: 2.25211238861084 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13591 , TFLOPS: 99.37303411649464, Tokens per sec: 81200.26426605994, Loss: 2.262188673019409 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13592 , TFLOPS: 98.82411728872248, Tokens per sec: 80751.73019571109, Loss: 2.24756121635437 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13593 , TFLOPS: 100.69656972761828, Tokens per sec: 82281.75928474673, Loss: 2.2198867797851562 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13594 , TFLOPS: 100.07007949172295, Tokens per sec: 81769.83798570326, Loss: 2.2467286586761475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13595 , TFLOPS: 99.0650405875928, Tokens per sec: 80948.59482513549, Loss: 2.266718626022339 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13596 , TFLOPS: 100.07295579220022, Tokens per sec: 81772.1882848658, Loss: 2.2605226039886475 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13597 , TFLOPS: 99.42073302595767, Tokens per sec: 81239.24027286135, Loss: 2.271873712539673 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13598 , TFLOPS: 98.37719159469103, Tokens per sec: 80386.53570622712, Loss: 2.237873077392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13599 , TFLOPS: 101.3685462604896, Tokens per sec: 82830.84860797074, Loss: 2.2433111667633057 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13600 , TFLOPS: 100.03229358562659, Tokens per sec: 81738.96214913616, Loss: 2.253679037094116 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/model.pt +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/optimizer.pt +[rank0]:[2024-08-29 16:25:00,055] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00781859501148574, 'preprocessing_with_comm': 0.0019342540035722777, 'state_converting': 2.3940204849932343, : 2.405405466008233}) +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013600, took 14.59s +Deleting checkpoint /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0011600 + eval ppl=7.601987361907959, eval loss=2.028409719467163 +------------------------------------------------------------------ +iteration: 13601 , TFLOPS: 98.65265999997021, Tokens per sec: 80611.62803136834, Loss: 2.24287748336792 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13602 , TFLOPS: 97.60031134322547, Tokens per sec: 79751.72685407782, Loss: 2.239943504333496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13603 , TFLOPS: 100.6745612274508, Tokens per sec: 82263.77557271069, Loss: 2.2756757736206055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13604 , TFLOPS: 98.39207076923624, Tokens per sec: 80398.69386277205, Loss: 2.2500810623168945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13605 , TFLOPS: 99.0205046348474, Tokens per sec: 80912.2033516898, Loss: 2.251939058303833 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13606 , TFLOPS: 101.23947325935794, Tokens per sec: 82725.37973611141, Loss: 2.249222755432129 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13607 , TFLOPS: 98.88681714746306, Tokens per sec: 80802.96386432598, Loss: 2.2553703784942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13608 , TFLOPS: 100.10275130914295, Tokens per sec: 81796.53496876455, Loss: 2.2584421634674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13609 , TFLOPS: 99.97363653387988, Tokens per sec: 81691.03196218709, Loss: 2.2705307006835938 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13610 , TFLOPS: 100.649880459334, Tokens per sec: 82243.60828174296, Loss: 2.2411935329437256 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13611 , TFLOPS: 100.74898660616, Tokens per sec: 82324.59046553366, Loss: 2.2431352138519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13612 , TFLOPS: 99.37596107794486, Tokens per sec: 81202.65596160761, Loss: 2.249138116836548 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13613 , TFLOPS: 99.92456996519559, Tokens per sec: 81650.9384058293, Loss: 2.255540370941162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13614 , TFLOPS: 99.24640881742506, Tokens per sec: 81096.79547456502, Loss: 2.255594253540039 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13615 , TFLOPS: 101.33030120332937, Tokens per sec: 82799.59758724975, Loss: 2.261274814605713 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13616 , TFLOPS: 100.66322684627846, Tokens per sec: 82254.51395808184, Loss: 2.2491133213043213 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13617 , TFLOPS: 98.88395034606062, Tokens per sec: 80800.62132710201, Loss: 2.253641366958618 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13618 , TFLOPS: 99.07893771490606, Tokens per sec: 80959.95052560703, Loss: 2.261931896209717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13619 , TFLOPS: 100.64889373140619, Tokens per sec: 82242.8020009525, Loss: 2.277974843978882 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13620 , TFLOPS: 100.14159381648841, Tokens per sec: 81828.2741814116, Loss: 2.2471113204956055 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13621 , TFLOPS: 100.68975617446739, Tokens per sec: 82276.19175507069, Loss: 2.231649398803711 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13622 , TFLOPS: 98.90649317656603, Tokens per sec: 80819.04165421205, Loss: 2.2473556995391846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13623 , TFLOPS: 99.41483483909803, Tokens per sec: 81234.4207125455, Loss: 2.2458646297454834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13624 , TFLOPS: 99.64406146513241, Tokens per sec: 81421.7276894965, Loss: 2.2544281482696533 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13625 , TFLOPS: 101.20689491679077, Tokens per sec: 82698.75913375863, Loss: 2.2535452842712402 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13626 , TFLOPS: 99.42373041498993, Tokens per sec: 81241.68951659928, Loss: 2.277038097381592 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13627 , TFLOPS: 99.53828250859688, Tokens per sec: 81335.29298111874, Loss: 2.2590138912200928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13628 , TFLOPS: 100.69006181945197, Tokens per sec: 82276.44150545559, Loss: 2.2707982063293457 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13629 , TFLOPS: 98.73599173894587, Tokens per sec: 80679.72054043516, Loss: 2.253999710083008 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13630 , TFLOPS: 98.68353409966255, Tokens per sec: 80636.856052997, Loss: 2.247685432434082 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13631 , TFLOPS: 98.99557970816522, Tokens per sec: 80891.8365524731, Loss: 2.269652843475342 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13632 , TFLOPS: 101.26997536916555, Tokens per sec: 82750.30379522938, Loss: 2.2505030632019043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13633 , TFLOPS: 97.62371262084307, Tokens per sec: 79770.84864042164, Loss: 2.271362781524658 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13634 , TFLOPS: 100.59390630718946, Tokens per sec: 82197.87036112278, Loss: 2.2543601989746094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13635 , TFLOPS: 99.4046837717302, Tokens per sec: 81226.12601408754, Loss: 2.2559032440185547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13636 , TFLOPS: 100.15256543978806, Tokens per sec: 81837.23937724446, Loss: 2.266991376876831 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13637 , TFLOPS: 98.98904571999842, Tokens per sec: 80886.49746254219, Loss: 2.262547254562378 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13638 , TFLOPS: 99.5451266761692, Tokens per sec: 81340.88552662662, Loss: 2.269078016281128 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13639 , TFLOPS: 99.53676317261404, Tokens per sec: 81334.05149257598, Loss: 2.2684988975524902 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13640 , TFLOPS: 98.49721549967276, Tokens per sec: 80484.61032867788, Loss: 2.2755930423736572 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13641 , TFLOPS: 99.36335762945406, Tokens per sec: 81192.35735940414, Loss: 2.258193016052246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13642 , TFLOPS: 98.50117419927622, Tokens per sec: 80487.84508402986, Loss: 2.259425401687622 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13643 , TFLOPS: 99.67056949503058, Tokens per sec: 81443.3880831037, Loss: 2.258667230606079 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13644 , TFLOPS: 100.66149036602559, Tokens per sec: 82253.09503536625, Loss: 2.2598962783813477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13645 , TFLOPS: 99.08772012653372, Tokens per sec: 80967.12686022738, Loss: 2.270082473754883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13646 , TFLOPS: 99.57484482570257, Tokens per sec: 81365.16899162358, Loss: 2.2590267658233643 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13647 , TFLOPS: 101.2422608888339, Tokens per sec: 82727.65757991618, Loss: 2.2551393508911133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13648 , TFLOPS: 100.7126896273695, Tokens per sec: 82294.93126979658, Loss: 2.2566843032836914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13649 , TFLOPS: 100.09248701447663, Tokens per sec: 81788.14774936624, Loss: 2.2683115005493164 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13650 , TFLOPS: 99.3536872933998, Tokens per sec: 81184.45547887763, Loss: 2.2588772773742676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13651 , TFLOPS: 100.61665431721813, Tokens per sec: 82216.4583457031, Loss: 2.249185562133789 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13652 , TFLOPS: 98.71072269717808, Tokens per sec: 80659.0725559237, Loss: 2.267585515975952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13653 , TFLOPS: 101.30641531781626, Tokens per sec: 82780.0798153196, Loss: 2.251532554626465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13654 , TFLOPS: 98.76543131568961, Tokens per sec: 80703.7763764346, Loss: 2.2579779624938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13655 , TFLOPS: 100.52855160486874, Tokens per sec: 82144.46735147682, Loss: 2.2677526473999023 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13656 , TFLOPS: 98.29298353590877, Tokens per sec: 80317.72713368788, Loss: 2.257591724395752 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13657 , TFLOPS: 100.64805551941599, Tokens per sec: 82242.11707635781, Loss: 2.258113384246826 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13658 , TFLOPS: 100.12702991466425, Tokens per sec: 81816.3736423229, Loss: 2.219724655151367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13659 , TFLOPS: 100.16402843214918, Tokens per sec: 81846.6060833864, Loss: 2.2385640144348145 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13660 , TFLOPS: 99.59694010520248, Tokens per sec: 81383.22361328603, Loss: 2.247556686401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13661 , TFLOPS: 100.13722190652746, Tokens per sec: 81824.70178123954, Loss: 2.2494308948516846 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13662 , TFLOPS: 99.56334657155719, Tokens per sec: 81355.77347216998, Loss: 2.283933162689209 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13663 , TFLOPS: 100.75652063186747, Tokens per sec: 82330.74671188231, Loss: 2.238381862640381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13664 , TFLOPS: 100.13891623220678, Tokens per sec: 81826.08625836794, Loss: 2.2715606689453125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13665 , TFLOPS: 98.38084490352536, Tokens per sec: 80389.5209189202, Loss: 2.2262935638427734 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13666 , TFLOPS: 101.36706084786815, Tokens per sec: 82829.63483907982, Loss: 2.249690532684326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13667 , TFLOPS: 99.33560613366717, Tokens per sec: 81169.68089780964, Loss: 2.2516896724700928 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13668 , TFLOPS: 100.11433519459725, Tokens per sec: 81806.00045976498, Loss: 2.2572622299194336 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13669 , TFLOPS: 99.293463302675, Tokens per sec: 81135.24490575293, Loss: 2.239131212234497 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13670 , TFLOPS: 100.65473130642683, Tokens per sec: 82247.57203377441, Loss: 2.2528045177459717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13671 , TFLOPS: 98.16953897007035, Tokens per sec: 80216.85740119546, Loss: 2.261331081390381 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13672 , TFLOPS: 100.78838179909599, Tokens per sec: 82356.78129180415, Loss: 2.263559341430664 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13673 , TFLOPS: 98.28831595635091, Tokens per sec: 80313.91313427695, Loss: 2.2555391788482666 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13674 , TFLOPS: 100.62436349749092, Tokens per sec: 82222.75771535594, Loss: 2.2661759853363037 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13675 , TFLOPS: 97.85233075441114, Tokens per sec: 79957.65840251441, Loss: 2.2648515701293945 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13676 , TFLOPS: 100.01145973735576, Tokens per sec: 81721.93827540311, Loss: 2.2561097145080566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13677 , TFLOPS: 99.10184774644425, Tokens per sec: 80978.67090213356, Loss: 2.2436797618865967 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13678 , TFLOPS: 98.56537200634132, Tokens per sec: 80540.30276478133, Loss: 2.237046003341675 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13679 , TFLOPS: 99.32421808897331, Tokens per sec: 81160.37543333592, Loss: 2.2745401859283447 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13680 , TFLOPS: 98.97719957158118, Tokens per sec: 80876.81766972347, Loss: 2.2482738494873047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13681 , TFLOPS: 100.2624322636916, Tokens per sec: 81927.0143872797, Loss: 2.2620904445648193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13682 , TFLOPS: 101.38699999568973, Tokens per sec: 82845.92763004417, Loss: 2.25877046585083 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13683 , TFLOPS: 98.87576149613837, Tokens per sec: 80793.93000703078, Loss: 2.2803077697753906 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13684 , TFLOPS: 100.72785080455604, Tokens per sec: 82307.31985795913, Loss: 2.239945650100708 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13685 , TFLOPS: 100.58439149880091, Tokens per sec: 82190.09556625555, Loss: 2.2301111221313477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13686 , TFLOPS: 100.67123247408303, Tokens per sec: 82261.05556264415, Loss: 2.252467155456543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13687 , TFLOPS: 100.04251064355373, Tokens per sec: 81747.31077017891, Loss: 2.2493298053741455 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13688 , TFLOPS: 98.7432604161918, Tokens per sec: 80685.65996372521, Loss: 2.232038736343384 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13689 , TFLOPS: 101.31352448768575, Tokens per sec: 82785.88890102628, Loss: 2.2736220359802246 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13690 , TFLOPS: 98.62814550365476, Tokens per sec: 80591.59660536972, Loss: 2.245795249938965 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13691 , TFLOPS: 101.28956868259503, Tokens per sec: 82766.31399600937, Loss: 2.2507455348968506 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13692 , TFLOPS: 98.11488916928842, Tokens per sec: 80172.20164216553, Loss: 2.273646354675293 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13693 , TFLOPS: 101.25235191187402, Tokens per sec: 82735.9032146083, Loss: 2.2787036895751953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13694 , TFLOPS: 98.19959579571858, Tokens per sec: 80241.41760716416, Loss: 2.2610230445861816 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13695 , TFLOPS: 100.67319047034256, Tokens per sec: 82262.65549179111, Loss: 2.248321771621704 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13696 , TFLOPS: 99.49198614342833, Tokens per sec: 81297.46302936506, Loss: 2.2519118785858154 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13697 , TFLOPS: 98.97021560171194, Tokens per sec: 80871.11088815994, Loss: 2.2423834800720215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13698 , TFLOPS: 99.50772569728122, Tokens per sec: 81310.3242239904, Loss: 2.260615348815918 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13699 , TFLOPS: 100.85051781320367, Tokens per sec: 82407.55422845483, Loss: 2.275999069213867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13700 , TFLOPS: 99.03583573966363, Tokens per sec: 80924.73079209289, Loss: 2.25400447845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13701 , TFLOPS: 100.75624727138768, Tokens per sec: 82330.52334199735, Loss: 2.262061595916748 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13702 , TFLOPS: 100.10048650735399, Tokens per sec: 81794.6843409216, Loss: 2.2804296016693115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13703 , TFLOPS: 99.02832845654896, Tokens per sec: 80918.59639780523, Loss: 2.2714505195617676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13704 , TFLOPS: 101.30469422980242, Tokens per sec: 82778.67346999874, Loss: 2.2574565410614014 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13705 , TFLOPS: 99.3047520336593, Tokens per sec: 81144.469218438, Loss: 2.268582582473755 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13706 , TFLOPS: 99.28699207261032, Tokens per sec: 81129.9570970828, Loss: 2.2355666160583496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13707 , TFLOPS: 99.46930301529768, Tokens per sec: 81278.92806145383, Loss: 2.2720601558685303 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13708 , TFLOPS: 100.21439343377232, Tokens per sec: 81887.76062272316, Loss: 2.2650630474090576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13709 , TFLOPS: 99.3872122854487, Tokens per sec: 81211.84961289084, Loss: 2.2336087226867676 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13710 , TFLOPS: 99.26768171116153, Tokens per sec: 81114.17810365009, Loss: 2.258242607116699 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13711 , TFLOPS: 98.4000220773025, Tokens per sec: 80405.19108127787, Loss: 2.241912364959717 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13712 , TFLOPS: 100.66196335350864, Tokens per sec: 82253.4815256144, Loss: 2.2517518997192383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13713 , TFLOPS: 97.6380171320895, Tokens per sec: 79782.53722479196, Loss: 2.2554421424865723 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13714 , TFLOPS: 100.65685942048603, Tokens per sec: 82249.31097055471, Loss: 2.259418249130249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13715 , TFLOPS: 97.60542959212938, Tokens per sec: 79755.90910701265, Loss: 2.2617106437683105 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13716 , TFLOPS: 98.57316679895712, Tokens per sec: 80546.67209048351, Loss: 2.2682948112487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13717 , TFLOPS: 99.96735554213414, Tokens per sec: 81685.89960214453, Loss: 2.2585158348083496 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13718 , TFLOPS: 99.62285289057212, Tokens per sec: 81404.397617266, Loss: 2.228976249694824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13719 , TFLOPS: 100.2102607402336, Tokens per sec: 81884.38369244766, Loss: 2.2368109226226807 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13720 , TFLOPS: 100.49967561050916, Tokens per sec: 82120.87203315138, Loss: 2.246023654937744 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13721 , TFLOPS: 100.73845592051416, Tokens per sec: 82315.98557120848, Loss: 2.2570250034332275 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13722 , TFLOPS: 100.77600727855697, Tokens per sec: 82346.66975252322, Loss: 2.2732372283935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13723 , TFLOPS: 100.67419509684251, Tokens per sec: 82263.4763979656, Loss: 2.2252373695373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13724 , TFLOPS: 98.96359554621202, Tokens per sec: 80865.70147039556, Loss: 2.241462469100952 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13725 , TFLOPS: 99.94965408389034, Tokens per sec: 81671.43528493746, Loss: 2.251955032348633 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13726 , TFLOPS: 98.87932024820144, Tokens per sec: 80796.83795495194, Loss: 2.2386276721954346 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13727 , TFLOPS: 100.50204889812406, Tokens per sec: 82122.81130755538, Loss: 2.2587950229644775 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13728 , TFLOPS: 99.50456907207108, Tokens per sec: 81307.74486427243, Loss: 2.2629735469818115 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13729 , TFLOPS: 100.0842095226362, Tokens per sec: 81781.38399770184, Loss: 2.247361898422241 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13730 , TFLOPS: 98.93945847839362, Tokens per sec: 80845.97845093775, Loss: 2.2691657543182373 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13731 , TFLOPS: 101.21736112222193, Tokens per sec: 82707.31133964052, Loss: 2.269226312637329 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13732 , TFLOPS: 97.72281675039518, Tokens per sec: 79851.82917584591, Loss: 2.2646303176879883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13733 , TFLOPS: 100.10545722359763, Tokens per sec: 81798.74604112189, Loss: 2.2476422786712646 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13734 , TFLOPS: 100.69796956568328, Tokens per sec: 82282.90312846468, Loss: 2.2608642578125 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13735 , TFLOPS: 99.07703296847802, Tokens per sec: 80958.39410826814, Loss: 2.239649772644043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13736 , TFLOPS: 100.13540224539904, Tokens per sec: 81823.21488928927, Loss: 2.246997833251953 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13737 , TFLOPS: 99.9805949044135, Tokens per sec: 81696.7178258745, Loss: 2.243070602416992 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13738 , TFLOPS: 100.07798849791713, Tokens per sec: 81776.30063826045, Loss: 2.2709600925445557 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13739 , TFLOPS: 101.14877212981084, Tokens per sec: 82651.2655083037, Loss: 2.2299182415008545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13740 , TFLOPS: 99.59278648980478, Tokens per sec: 81379.829587221, Loss: 2.260667324066162 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13741 , TFLOPS: 99.38015103339714, Tokens per sec: 81206.07968206672, Loss: 2.2880148887634277 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13742 , TFLOPS: 100.86031979800154, Tokens per sec: 82415.56368255871, Loss: 2.241102695465088 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13743 , TFLOPS: 98.69308165137176, Tokens per sec: 80644.65760327442, Loss: 2.2573604583740234 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13744 , TFLOPS: 99.97272670537669, Tokens per sec: 81690.28851789606, Loss: 2.2753002643585205 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13745 , TFLOPS: 98.8171513531334, Tokens per sec: 80746.03815042211, Loss: 2.2537927627563477 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13746 , TFLOPS: 100.64313765115413, Tokens per sec: 82238.09855960387, Loss: 2.2799384593963623 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13747 , TFLOPS: 98.08176162626843, Tokens per sec: 80145.13227398507, Loss: 2.240691900253296 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13748 , TFLOPS: 100.24980032550853, Tokens per sec: 81916.6925054153, Loss: 2.2283153533935547 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13749 , TFLOPS: 98.87450168192238, Tokens per sec: 80792.9005804044, Loss: 2.231468439102173 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13750 , TFLOPS: 100.64424469716644, Tokens per sec: 82239.00315539842, Loss: 2.2474923133850098 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13751 , TFLOPS: 98.39562299535842, Tokens per sec: 80401.59647817892, Loss: 2.2649428844451904 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13752 , TFLOPS: 100.57509419321502, Tokens per sec: 82182.49849852738, Loss: 2.2406930923461914 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13753 , TFLOPS: 98.40273751655404, Tokens per sec: 80407.40993659192, Loss: 2.2618777751922607 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13754 , TFLOPS: 99.27745927040546, Tokens per sec: 81122.16759900507, Loss: 2.2186038494110107 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13755 , TFLOPS: 100.67970795907844, Tokens per sec: 82267.98109961228, Loss: 2.2644715309143066 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13756 , TFLOPS: 98.33694659089592, Tokens per sec: 80353.65047762758, Loss: 2.264904499053955 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13757 , TFLOPS: 100.23127403775752, Tokens per sec: 81901.55419878487, Loss: 2.26090669631958 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13758 , TFLOPS: 101.22943395859298, Tokens per sec: 82717.17636502178, Loss: 2.2583494186401367 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13759 , TFLOPS: 99.9347365970902, Tokens per sec: 81659.24582246278, Loss: 2.24082612991333 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13760 , TFLOPS: 100.59073478579803, Tokens per sec: 82195.27882935164, Loss: 2.2547459602355957 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13761 , TFLOPS: 100.53641557623143, Tokens per sec: 82150.8932049139, Loss: 2.245553731918335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13762 , TFLOPS: 99.31530330426955, Tokens per sec: 81153.09093326751, Loss: 2.270256280899048 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13763 , TFLOPS: 100.71092436585768, Tokens per sec: 82293.48882917341, Loss: 2.248887062072754 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13764 , TFLOPS: 97.56008163280497, Tokens per sec: 79718.85412208835, Loss: 2.2467281818389893 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13765 , TFLOPS: 101.29805102529562, Tokens per sec: 82773.24513658494, Loss: 2.2749111652374268 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13766 , TFLOPS: 98.87485489397514, Tokens per sec: 80793.1891990652, Loss: 2.251370668411255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13767 , TFLOPS: 100.715973633511, Tokens per sec: 82297.61471575264, Loss: 2.2553210258483887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13768 , TFLOPS: 99.3613303803883, Tokens per sec: 81190.70084200648, Loss: 2.269416332244873 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13769 , TFLOPS: 100.64291533519345, Tokens per sec: 82237.91689950945, Loss: 2.2583792209625244 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13770 , TFLOPS: 98.16050782601198, Tokens per sec: 80209.47782090305, Loss: 2.285550355911255 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13771 , TFLOPS: 100.12856241594856, Tokens per sec: 81817.62588857241, Loss: 2.246037483215332 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13772 , TFLOPS: 99.58387345398712, Tokens per sec: 81372.54651621245, Loss: 2.2674078941345215 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13773 , TFLOPS: 99.12549560974732, Tokens per sec: 80997.99417998866, Loss: 2.2361741065979004 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13774 , TFLOPS: 99.25691770072848, Tokens per sec: 81105.38255363496, Loss: 2.2494895458221436 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13775 , TFLOPS: 100.18852171484227, Tokens per sec: 81866.62017518813, Loss: 2.2470719814300537 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13776 , TFLOPS: 100.13824086451523, Tokens per sec: 81825.53439804167, Loss: 2.269695997238159 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13777 , TFLOPS: 100.77775436617466, Tokens per sec: 82348.09734278967, Loss: 2.254572868347168 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13778 , TFLOPS: 100.08762019281444, Tokens per sec: 81784.17094410294, Loss: 2.2632250785827637 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13779 , TFLOPS: 99.5663700267164, Tokens per sec: 81358.24401521124, Loss: 2.267454147338867 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13780 , TFLOPS: 100.68708009739196, Tokens per sec: 82274.00506360398, Loss: 2.2470359802246094 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13781 , TFLOPS: 99.09046939798858, Tokens per sec: 80969.3733607057, Loss: 2.256946325302124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13782 , TFLOPS: 98.9932417822628, Tokens per sec: 80889.92617303466, Loss: 2.266441822052002 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13783 , TFLOPS: 99.29245735636206, Tokens per sec: 81134.42292112528, Loss: 2.275287389755249 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13784 , TFLOPS: 100.67835398673223, Tokens per sec: 82266.87473395387, Loss: 2.2424886226654053 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13785 , TFLOPS: 99.46608841711546, Tokens per sec: 81276.30133052802, Loss: 2.240109920501709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13786 , TFLOPS: 100.11153827877604, Tokens per sec: 81803.71502784846, Loss: 2.2500953674316406 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13787 , TFLOPS: 96.39885961428827, Tokens per sec: 78769.98971824428, Loss: 2.2605762481689453 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13788 , TFLOPS: 100.61975602512203, Tokens per sec: 82218.99283107642, Loss: 2.2579874992370605 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13789 , TFLOPS: 97.22105664251633, Tokens per sec: 79441.82807524384, Loss: 2.2529726028442383 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13790 , TFLOPS: 101.34698316454693, Tokens per sec: 82813.2288471925, Loss: 2.272580623626709 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13791 , TFLOPS: 98.3432285323723, Tokens per sec: 80358.78361371948, Loss: 2.266329765319824 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13792 , TFLOPS: 98.58058998055282, Tokens per sec: 80552.73776325505, Loss: 2.238492727279663 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13793 , TFLOPS: 100.59528144787026, Tokens per sec: 82198.99402397234, Loss: 2.26275897026062 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13794 , TFLOPS: 97.79665438687309, Tokens per sec: 79912.16380936187, Loss: 2.242032766342163 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13795 , TFLOPS: 100.65850775503378, Tokens per sec: 82250.6578671456, Loss: 2.26542067527771 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13796 , TFLOPS: 101.12525578461009, Tokens per sec: 82632.04969727558, Loss: 2.2580831050872803 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13797 , TFLOPS: 99.44697945215702, Tokens per sec: 81260.68690335196, Loss: 2.2477712631225586 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13798 , TFLOPS: 101.28629249699578, Tokens per sec: 82763.6369404197, Loss: 2.241546630859375 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13799 , TFLOPS: 99.53679500638013, Tokens per sec: 81334.07750476572, Loss: 2.26208758354187 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13800 , TFLOPS: 100.04630427824867, Tokens per sec: 81750.41064674497, Loss: 2.257981300354004 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800 +Saving model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/model.pt +Saved model state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/model.pt +[rank0]:[2024-08-29 17:19:00,967] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.00774207100039348, 'preprocessing_with_comm': 0.0018419440020807087, 'state_converting': 2.4383058559906203, : 2.4495451700058766}) +Saving optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/optimizer.pt +Saved optimizer state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/scheduler.pt +Saving RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/rng.pt +Saved RNG states to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800/rng.pt +Saved checkpoint to /work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5/iter_0013800, took 14.74s +Traceback (most recent call last): + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_errors.py", line 304, in hf_raise_for_status + response.raise_for_status() + File "/singularity_home/.local/lib/python3.10/site-packages/requests/models.py", line 1024, in raise_for_status + raise HTTPError(http_error_msg, response=self) +requests.exceptions.HTTPError: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/koichi12/yans-baseline-qwen2-1.5B-3.5e-5/commit/main +The above exception was the direct cause of the following exception: +Traceback (most recent call last): + File "/project/examples/finetuning.py", line 13, in + main() + File "/project/src/llama_recipes/finetuning.py", line 282, in main + train( + File "/project/src/llama_recipes/utils/train_utils.py", line 162, in train + save_checkpoint( + File "/project/src/llama_recipes/utils/checkpoint.py", line 182, in save_checkpoint + upload_checkpoint_to_huggingface_hub( + File "/project/src/llama_recipes/utils/hf_hub_utils.py", line 99, in upload_checkpoint_to_huggingface_hub + fs.put( + File "/usr/local/lib/python3.10/dist-packages/fsspec/spec.py", line 1057, in put + self.put_file(lpath, rpath, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/fsspec/spec.py", line 985, in put_file + with self.open(rpath, "wb", **kwargs) as f2: + File "/usr/local/lib/python3.10/dist-packages/fsspec/spec.py", line 1963, in __exit__ + self.close() + File "/usr/local/lib/python3.10/dist-packages/fsspec/spec.py", line 1930, in close + self.flush(force=True) + File "/usr/local/lib/python3.10/dist-packages/fsspec/spec.py", line 1801, in flush + if self._upload_chunk(final=force) is not False: + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_file_system.py", line 740, in _upload_chunk + self.fs._api.upload_file( + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 1398, in _inner + return fn(self, *args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 4507, in upload_file + commit_info = self.create_commit( + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_validators.py", line 114, in _inner_fn + return fn(*args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 1398, in _inner + return fn(self, *args, **kwargs) + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/hf_api.py", line 3852, in create_commit + hf_raise_for_status(commit_resp, endpoint_name="commit") + File "/usr/local/lib/python3.10/dist-packages/huggingface_hub/utils/_errors.py", line 371, in hf_raise_for_status + raise HfHubHTTPError(str(e), response=response) from e +huggingface_hub.utils._errors.HfHubHTTPError: 502 Server Error: Bad Gateway for url: https://huggingface.co/api/models/koichi12/yans-baseline-qwen2-1.5B-3.5e-5/commit/main \ No newline at end of file diff --git a/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt b/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20240828_213016-pkqwjpaw/files/wandb-metadata.json b/wandb/run-20240828_213016-pkqwjpaw/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4a82b26759511f71f14a6fa1a19753d550cbc5fa --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/files/wandb-metadata.json @@ -0,0 +1,880 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-08-28T12:30:17.069615", + "startedAt": "2024-08-28T12:30:16.374560", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "16", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "1280", + "--train-iters", + "23178", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "--train-data-path", + "1754785366", + "/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document", + "28623823675", + "/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document", + "--valid-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--test-data-path", + "1205770", + "/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document", + "--lr", + "3.5e-5", + "--min-lr", + "3.5e-6", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "500", + "--lr-decay-iters", + "23178", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/Qwen/Qwen2-1.5B", + "--save", + "/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5", + "--load", + "/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5", + "--fsdp-activation-checkpointing", + "--num-workers", + "4", + "--sharding-strategy", + "FULL_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/yans-baseline-qwen2-1.5B-3.5e-5", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "yans_experiment", + "--wandb-name", + "yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-28-21:29:36" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "887a2cc5d104c10264701f95cbbb0a6a116768d6" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 144, + "cpu_count_logical": 144, + "cpu_freq": { + "current": 2400.0389999999943, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.039, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 8, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 453.4449653625488 + } +} diff --git a/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json b/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..d76d4c5a42b2d3dc4b7e6d4d50fc54567c920224 --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.257981300354004, "training/perplexity": 9.56376330117787, "utils/batch_size": 16, "utils/global_batch_size": 1280, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 10, "utils/iteration": 13800, "optimizer/lr": 1.5023836642503739e-05, "optimizer/variance_l2": 0.0004649567327322915, "optimizer/variance_sqrt_l2": 0.15750536869753576, "optimizer/momentum_l2": 0.03505431072031769, "optimizer/weight_l2": 1232.9854013734307, "optimizer/variance_l1": 0.02482891082763672, "optimizer/variance_sqrt_l1": 943.875, "optimizer/momentum_l1": 173.140625, "optimizer/weight_l1": 3161088.0, "optimizer/variance_abs_max": 0.0003414154052734375, "optimizer/variance_sqrt_abs_max": 0.0184326171875, "optimizer/momentum_abs_max": 0.00323486328125, "optimizer/weight_abs_max": 432.0, "stats/1_iteration_time": 16.048849047001568, "stats/tokens_per_sec": 81750.41064674497, "stats/tokens_per_sec_per_gpu": 10218.801330843122, "stats/tflops": 100.04630427824867, "_timestamp": 1724919531.9077263, "_runtime": 71315.51744937897, "_step": 13800, "evaluation/val_loss": 2.028409719467163, "evaluation/val_ppl": 7.601987361907959, "_wandb": {"runtime": 71460}} \ No newline at end of file diff --git a/wandb/run-20240828_213016-pkqwjpaw/logs/debug-internal.log b/wandb/run-20240828_213016-pkqwjpaw/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..e891e73075079aa5858f3155e0e7cfc39b912752 --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/logs/debug-internal.log @@ -0,0 +1,57076 @@ +2024-08-28 21:30:16,391 INFO StreamThr :12190 [internal.py:wandb_internal():86] W&B internal server running at pid: 12190, started at: 2024-08-28 21:30:16.390571 +2024-08-28 21:30:16,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status +2024-08-28 21:30:16,397 INFO WriterThread:12190 [datastore.py:open_for_write():87] open: /project/wandb/run-20240828_213016-pkqwjpaw/run-pkqwjpaw.wandb +2024-08-28 21:30:16,398 DEBUG SenderThread:12190 [sender.py:send():382] send: header +2024-08-28 21:30:16,493 DEBUG SenderThread:12190 [sender.py:send():382] send: run +2024-08-28 21:30:16,919 INFO SenderThread:12190 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20240828_213016-pkqwjpaw/files +2024-08-28 21:30:16,919 INFO SenderThread:12190 [sender.py:_start_run_threads():1136] run started: pkqwjpaw with start time 1724848216.390277 +2024-08-28 21:30:16,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: check_version +2024-08-28 21:30:16,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: check_version +2024-08-28 21:30:16,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: run_start +2024-08-28 21:30:17,001 DEBUG HandlerThread:12190 [system_info.py:__init__():27] System info init +2024-08-28 21:30:17,001 DEBUG HandlerThread:12190 [system_info.py:__init__():42] System info init done +2024-08-28 21:30:17,001 INFO HandlerThread:12190 [system_monitor.py:start():194] Starting system monitor +2024-08-28 21:30:17,001 INFO SystemMonitor:12190 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-08-28 21:30:17,001 INFO HandlerThread:12190 [system_monitor.py:probe():214] Collecting system info +2024-08-28 21:30:17,002 INFO SystemMonitor:12190 [interfaces.py:start():190] Started cpu monitoring +2024-08-28 21:30:17,002 INFO SystemMonitor:12190 [interfaces.py:start():190] Started disk monitoring +2024-08-28 21:30:17,003 INFO SystemMonitor:12190 [interfaces.py:start():190] Started gpu monitoring +2024-08-28 21:30:17,003 INFO SystemMonitor:12190 [interfaces.py:start():190] Started memory monitoring +2024-08-28 21:30:17,005 INFO SystemMonitor:12190 [interfaces.py:start():190] Started network monitoring +2024-08-28 21:30:17,069 DEBUG HandlerThread:12190 [system_info.py:probe():151] Probing system +2024-08-28 21:30:17,071 DEBUG HandlerThread:12190 [system_info.py:_probe_git():136] Probing git +2024-08-28 21:30:17,085 DEBUG HandlerThread:12190 [system_info.py:_probe_git():144] Probing git done +2024-08-28 21:30:17,085 DEBUG HandlerThread:12190 [system_info.py:probe():199] Probing system done +2024-08-28 21:30:17,085 DEBUG HandlerThread:12190 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-08-28T12:30:17.069615', 'startedAt': '2024-08-28T12:30:16.374560', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '16', '--valid_micro_batch_size', '1', '--global-batch-size', '1280', '--train-iters', '23178', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/Qwen/Qwen2-1.5B', '--train-data-path', '1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document', '--valid-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--test-data-path', '1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document', '--lr', '3.5e-5', '--min-lr', '3.5e-6', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '500', '--lr-decay-iters', '23178', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/Qwen/Qwen2-1.5B', '--save', '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', '--load', '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', '--fsdp-activation-checkpointing', '--num-workers', '4', '--sharding-strategy', 'FULL_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/yans-baseline-qwen2-1.5B-3.5e-5', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'yans_experiment', '--wandb-name', 'yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-28-21:29:36'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '887a2cc5d104c10264701f95cbbb0a6a116768d6'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 144, 'cpu_count_logical': 144, 'cpu_freq': {'current': 2400.0389999999943, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}, {'current': 2400.039, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 453.4449653625488}} +2024-08-28 21:30:17,085 INFO HandlerThread:12190 [system_monitor.py:probe():224] Finished collecting system info +2024-08-28 21:30:17,085 INFO HandlerThread:12190 [system_monitor.py:probe():227] Publishing system info +2024-08-28 21:30:17,088 INFO HandlerThread:12190 [system_monitor.py:probe():229] Finished publishing system info +2024-08-28 21:30:17,114 DEBUG SenderThread:12190 [sender.py:send():382] send: files +2024-08-28 21:30:17,114 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-08-28 21:30:17,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: python_packages +2024-08-28 21:30:17,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:30:17,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:30:17,127 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: python_packages +2024-08-28 21:30:17,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:30:17,377 DEBUG SenderThread:12190 [sender.py:send():382] send: telemetry +2024-08-28 21:30:17,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt +2024-08-28 21:30:17,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-metadata.json +2024-08-28 21:30:18,168 INFO wandb-upload_0:12190 [upload_job.py:push():131] Uploaded file /tmp/tmp7te__ve_wandb/ordac91w-wandb-metadata.json +2024-08-28 21:30:19,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:30:21,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:30:22,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:27,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:27,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:30:32,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:30:32,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:30:32,126 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:30:33,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:38,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:43,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:47,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:30:47,126 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:30:47,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:30:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:48,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml +2024-08-28 21:30:53,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:30:58,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:02,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:31:02,126 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:31:02,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:31:04,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:05,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:31:07,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:31:09,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:09,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:31:14,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:17,005 DEBUG SystemMonitor:12190 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-08-28 21:31:17,010 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:31:17,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:31:17,126 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:31:17,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:31:20,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:23,645 DEBUG SenderThread:12190 [sender.py:send():382] send: config +2024-08-28 21:31:23,645 DEBUG SenderThread:12190 [sender.py:send():382] send: config +2024-08-28 21:31:25,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:25,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:31:30,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:32,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:31:32,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:31:32,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:31:36,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:41,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:46,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:47,014 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:31:47,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:31:47,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:31:47,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:31:52,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:31:52,984 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml +2024-08-28 21:31:57,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:02,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:32:02,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:32:02,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:32:02,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:07,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:09,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:32:11,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:32:13,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:17,017 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:32:17,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:32:17,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:32:17,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:32:18,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:23,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:25,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:32:25,402 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:32:25,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:32:25,405 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:32:26,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:32:28,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:32:29,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:32,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:32:32,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:32:32,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:32:34,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:39,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:41,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:32:41,504 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:32:41,504 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:32:41,506 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:32:42,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:32:44,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:32:45,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:47,020 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:32:47,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:32:47,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:32:47,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:32:50,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:55,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:32:57,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:32:57,492 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:32:57,493 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:32:57,494 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:32:58,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:33:00,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:33:01,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:02,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:33:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:33:02,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:33:06,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:11,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:13,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:33:13,577 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:33:13,578 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:33:13,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:33:14,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:33:16,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:33:17,024 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:33:17,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:17,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:33:17,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:33:17,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:33:22,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:27,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:29,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:33:29,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:33:29,556 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:33:29,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:33:30,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:33:32,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:33:32,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:33:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:33:32,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:33:32,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:37,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:42,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:45,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:33:45,523 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:33:45,524 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:33:45,526 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:33:46,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:33:47,026 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:33:47,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:33:47,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:33:47,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:33:48,069 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:33:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:53,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:33:58,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:01,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:34:01,372 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:34:01,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:34:01,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:34:02,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:34:02,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:34:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:34:02,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:34:04,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:34:04,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:09,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:14,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:17,031 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:34:17,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:34:17,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:34:17,331 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:34:17,331 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:34:17,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:34:17,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:34:17,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:34:18,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:34:19,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:20,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:34:24,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:29,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:32,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:34:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:34:32,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:34:33,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:34:33,284 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:34:33,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:34:33,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:34:34,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:34:35,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:36,107 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:34:40,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:45,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:47,033 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:34:47,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:34:47,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:34:47,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:34:49,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:34:49,140 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:34:49,141 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:34:49,142 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:34:50,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:34:50,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:34:51,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:34:56,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:01,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:02,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:35:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:35:02,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:35:05,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:35:05,088 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:35:05,089 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:35:05,090 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:35:05,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:35:06,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:35:07,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:12,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:17,078 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:35:17,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:35:17,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:17,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:35:17,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:35:20,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:35:20,948 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:35:20,948 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:35:20,950 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:35:21,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:35:22,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:35:22,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:27,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:32,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:35:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:35:32,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:35:33,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:37,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:35:37,044 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:35:37,045 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:35:37,046 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:35:37,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:35:38,155 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:35:39,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:44,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:47,082 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:35:47,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:35:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:35:47,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:35:49,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:52,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:35:52,926 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:35:52,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:35:52,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:35:53,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:35:54,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:35:54,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:35:59,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:02,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:36:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:36:02,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:36:05,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:08,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:36:08,781 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:36:08,782 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:36:08,784 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:36:09,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:36:10,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:36:10,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:15,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:17,083 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:36:17,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:36:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:36:17,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:36:21,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:24,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:36:24,637 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:36:24,637 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:36:24,639 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:36:25,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:36:26,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:36:27,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:32,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:36:32,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:36:32,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:36:32,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:37,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:40,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:36:40,622 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:36:40,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:36:40,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:36:41,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:36:42,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:36:43,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:47,087 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:36:47,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:36:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:36:47,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:36:48,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:53,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:36:56,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:36:56,472 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:36:56,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:36:56,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:36:57,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:36:58,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:36:59,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:02,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:37:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:37:02,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:37:04,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:09,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:12,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:37:12,455 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:37:12,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:37:12,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:37:13,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:37:14,230 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:37:15,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:17,090 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:37:17,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:37:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:37:17,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:37:20,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:25,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:28,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:37:28,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:37:28,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:37:28,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:37:29,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:37:30,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:37:31,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:32,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:37:32,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:37:32,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:37:36,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:41,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:44,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:37:44,389 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:37:44,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:37:44,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:37:45,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:37:46,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:37:47,093 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:37:47,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:47,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:37:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:37:47,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:37:52,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:37:57,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:00,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:38:00,268 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:38:00,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:38:00,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:38:01,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:38:02,271 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:38:02,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:38:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:38:02,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:38:02,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:07,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:12,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:16,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:38:16,349 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:38:16,350 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:38:16,351 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:38:17,095 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:38:17,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:38:17,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:38:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:38:17,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:38:17,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:18,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:38:22,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:27,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:32,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:38:32,216 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:38:32,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:38:32,218 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:38:32,296 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:38:32,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:38:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:38:32,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:38:33,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:34,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:38:38,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:43,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:47,099 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:38:47,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:38:47,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:38:47,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:38:48,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:38:48,087 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:38:48,087 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:38:48,089 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:38:48,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:38:49,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:50,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:38:54,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:38:59,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:02,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:39:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:39:02,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:39:04,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:39:04,312 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:39:04,313 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:39:04,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:04,315 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:39:04,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:39:06,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:39:09,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:14,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:17,103 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:39:17,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:39:17,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:39:17,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:39:19,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:20,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:39:20,171 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:39:20,172 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:39:20,173 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:39:20,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:39:22,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:39:25,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:30,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:32,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:39:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:39:32,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:39:35,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:36,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:39:36,141 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:39:36,141 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:39:36,143 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:39:36,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:39:38,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:39:41,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:46,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:47,105 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:39:47,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:39:47,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:39:47,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:39:51,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:39:52,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:39:52,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:39:52,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:39:52,182 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:39:52,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:39:54,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:39:57,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:02,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:02,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:40:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:40:02,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:40:07,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:08,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:40:08,163 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:40:08,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:40:08,165 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:40:08,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:40:10,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:40:13,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:17,108 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:40:17,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:40:17,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:40:17,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:40:18,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:23,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:24,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:40:24,119 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:40:24,119 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:40:24,121 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:40:24,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:40:26,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:40:29,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:32,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:40:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:40:32,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:40:34,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:39,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:40,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:40:40,221 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:40:40,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:40:40,223 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:40:40,395 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:40:42,396 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:40:45,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:47,111 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:40:47,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:40:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:40:47,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:40:50,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:55,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:40:56,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:40:56,217 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:40:56,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:40:56,219 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:40:56,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:40:58,409 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:41:01,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:02,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:41:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:41:02,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:41:06,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:11,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:12,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:41:12,180 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:41:12,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:41:12,182 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:41:12,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:41:14,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:41:17,115 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:41:17,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:17,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:41:17,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:41:17,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:41:22,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:27,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:28,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:41:28,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:41:28,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:41:28,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:41:28,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:41:30,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:41:32,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:41:32,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:41:32,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:41:32,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:37,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:42,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:44,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:41:44,341 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:41:44,342 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:41:44,343 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:41:44,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:41:46,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:41:47,120 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:41:47,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:41:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:41:47,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:41:48,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:53,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:41:58,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:00,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:42:00,193 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:42:00,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:42:00,195 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:42:00,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:42:02,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:42:02,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:42:02,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:42:02,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:42:04,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:09,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:14,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:16,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:42:16,299 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:42:16,299 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:42:16,301 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:42:16,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:42:17,122 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:42:17,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:42:17,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:42:17,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:42:18,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:42:19,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:24,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:29,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:32,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:42:32,149 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:42:32,149 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:42:32,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:42:32,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:42:32,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:42:32,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:42:32,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:42:34,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:42:35,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:40,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:45,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:47,126 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:42:47,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:42:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:42:47,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:42:48,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:42:48,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:42:48,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:42:48,110 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:42:48,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:42:50,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:42:51,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:42:56,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:01,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:02,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:43:02,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:43:02,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:43:04,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:43:04,191 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:43:04,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:43:04,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:43:04,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:43:06,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:06,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:43:11,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:16,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:17,129 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:43:17,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:43:17,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:43:17,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:43:20,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:43:20,048 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:43:20,049 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:43:20,050 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:43:20,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:43:22,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:22,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:43:27,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:32,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:32,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:43:32,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:43:32,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:43:36,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:43:36,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:43:36,124 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:43:36,126 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:43:36,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:43:37,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:38,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:43:42,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:47,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:47,132 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:43:47,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:43:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:43:47,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:43:52,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:43:52,147 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:43:52,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:43:52,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:43:52,150 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:43:52,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:43:54,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:43:57,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:02,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:02,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:44:02,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:44:02,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:44:07,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:08,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:44:08,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:44:08,011 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:44:08,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:44:08,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:44:10,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:44:13,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:17,135 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:44:17,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:44:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:44:17,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:44:18,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:23,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:23,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:44:23,994 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:44:23,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:44:23,996 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:44:24,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:44:26,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:44:28,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:32,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:44:32,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:44:32,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:44:34,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:39,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:39,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:44:39,839 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:44:39,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:44:39,841 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:44:40,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:44:40,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:44:44,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:47,138 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:44:47,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:44:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:44:47,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:44:50,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:55,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:44:55,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:44:55,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:44:55,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:44:55,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:44:56,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:44:56,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:45:00,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:02,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:45:02,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:45:02,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:45:06,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:11,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:11,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:45:11,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:45:11,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:45:11,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:45:12,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:45:12,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:45:16,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:17,141 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:45:17,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:45:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:45:17,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:45:22,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:27,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:27,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:45:27,716 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:45:27,716 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:45:27,718 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:45:28,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:45:28,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:45:32,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:45:32,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:45:32,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:45:32,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:37,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:42,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:43,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:45:43,935 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:45:43,936 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:45:43,937 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:45:44,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:45:44,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:45:47,145 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:45:47,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:45:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:45:47,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:45:48,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:53,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:58,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:45:59,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:45:59,900 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:45:59,900 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:45:59,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:46:00,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:46:00,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:46:02,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:46:02,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:46:02,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:46:03,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:08,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:13,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:15,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:46:15,859 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:46:15,859 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:46:15,861 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:46:16,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:46:16,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:46:17,148 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:46:17,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:46:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:46:17,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:46:18,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:23,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:28,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:31,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:46:31,700 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:46:31,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:46:31,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:46:32,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:46:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:46:32,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:46:32,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:46:32,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:46:33,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:38,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:43,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:47,151 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:46:47,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:46:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:46:47,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:46:47,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:46:47,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:46:47,763 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:46:47,764 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:46:48,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:46:48,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:46:48,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:53,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:46:58,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:02,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:47:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:47:02,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:47:03,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:47:03,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:47:03,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:47:03,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:47:03,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:47:04,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:04,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:47:09,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:14,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:17,155 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:47:17,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:47:17,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:47:17,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:47:19,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:47:19,569 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:47:19,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:47:19,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:47:19,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:47:20,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:20,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:47:25,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:30,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:32,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:47:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:47:32,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:47:35,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:47:35,622 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:47:35,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:47:35,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:35,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:47:35,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:47:36,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:47:40,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:45,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:47,158 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:47:47,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:47:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:47:47,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:47:50,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:47:51,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:47:51,472 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:47:51,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:47:51,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:47:51,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:47:52,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:47:56,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:01,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:02,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:48:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:48:02,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:48:06,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:07,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:48:07,433 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:48:07,434 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:48:07,435 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:48:07,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:48:08,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:48:12,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:17,160 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:48:17,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:48:17,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:48:17,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:48:17,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:22,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:23,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:48:23,478 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:48:23,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:48:23,480 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:48:23,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:48:24,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:48:28,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:32,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:48:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:48:32,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:48:33,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:38,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:39,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:48:39,572 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:48:39,572 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:48:39,574 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:48:39,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:48:40,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:48:44,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:47,165 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:48:47,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:48:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:48:47,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:48:49,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:54,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:48:55,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:48:55,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:48:55,439 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:48:55,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:48:55,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:48:56,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:49:00,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:02,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:49:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:49:02,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:49:05,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:10,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:11,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:49:11,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:49:11,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:49:11,608 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:49:11,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:49:12,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:49:16,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:17,168 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:49:17,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:49:17,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:49:17,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:49:21,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:26,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:27,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:49:27,595 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:49:27,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:49:27,597 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:49:27,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:49:28,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:49:32,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:49:32,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:32,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:49:32,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:49:37,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:42,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:43,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:49:43,559 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:49:43,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:49:43,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:49:43,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:49:44,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:49:47,173 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:49:47,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:49:47,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:49:47,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:49:48,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:53,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:58,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:49:59,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:49:59,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:49:59,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:49:59,612 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:49:59,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:50:00,831 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:50:02,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:50:02,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:50:02,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:50:04,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:09,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:14,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:15,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:50:15,798 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:50:15,799 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:50:15,801 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:50:15,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:50:16,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:50:17,176 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:50:17,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:50:17,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:50:17,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:50:19,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:24,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:29,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:31,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:50:31,657 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:50:31,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:50:31,660 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:50:31,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:50:32,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:50:32,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:50:32,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:50:32,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:50:35,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:40,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:45,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:47,180 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:50:47,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:50:47,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:50:47,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:50:47,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:50:47,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:50:47,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:50:47,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:50:47,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:50:48,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:50:50,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:50:55,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:00,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:02,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:51:02,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:51:02,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:51:03,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:51:03,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:51:03,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:51:03,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:51:03,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:51:04,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:51:06,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:11,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:16,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:17,185 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:51:17,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:51:17,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:51:17,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:51:19,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:51:19,597 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:51:19,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:51:19,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:51:19,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:51:20,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:51:21,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:26,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:31,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:32,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:51:32,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:51:32,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:51:35,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:51:35,639 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:51:35,640 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:51:35,641 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:51:35,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:51:36,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:36,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:51:41,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:46,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:47,236 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:51:47,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:51:47,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:51:47,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:51:51,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:51:51,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:51:51,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:51:51,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:51:51,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:51:52,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:51:52,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:51:57,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:02,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:52:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:52:02,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:52:02,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:07,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:52:07,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:52:07,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:52:07,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:52:07,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:52:08,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:08,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:52:13,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:17,241 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:52:17,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:52:17,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:52:17,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:52:18,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:23,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:23,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:52:23,831 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:52:23,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:52:23,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:52:23,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:52:24,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:52:28,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:32,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:52:32,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:52:32,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:52:34,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:39,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:39,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:52:39,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:52:39,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:52:39,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:52:39,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:52:40,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:52:44,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:47,246 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:52:47,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:52:47,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:52:47,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:52:50,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:55,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:52:55,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:52:55,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:52:55,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:52:55,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:52:55,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:52:56,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:53:00,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:02,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:53:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:53:02,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:53:06,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:11,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:53:11,660 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:53:11,660 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:53:11,662 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:53:11,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:53:12,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:12,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:53:17,248 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:53:17,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:53:17,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:53:17,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:53:17,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:22,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:27,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:53:27,644 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:53:27,644 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:53:27,646 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:53:27,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:53:28,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:28,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:53:32,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:53:32,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:53:32,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:53:33,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:38,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:43,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:43,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:53:43,710 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:53:43,710 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:53:43,712 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:53:44,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:53:45,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:53:47,250 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:53:47,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:53:47,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:53:47,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:53:49,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:54,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:59,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:53:59,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:53:59,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:53:59,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:53:59,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:54:00,016 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:54:01,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:54:02,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:54:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:54:02,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:54:04,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:09,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:14,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:15,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:54:15,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:54:15,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:54:15,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:54:16,028 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:54:17,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:54:17,254 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:54:17,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:54:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:54:17,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:54:19,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:24,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:29,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:31,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:54:31,727 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:54:31,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:54:31,730 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:54:32,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:54:32,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:54:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:54:32,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:54:33,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:54:35,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:40,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:45,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:47,259 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:54:47,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:54:47,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:54:47,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:54:47,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:54:47,700 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:54:47,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:54:47,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:54:48,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:54:49,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:54:50,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:54:55,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:00,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:02,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:55:02,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:55:02,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:55:03,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:55:03,518 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:55:03,519 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:55:03,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:55:04,064 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:55:05,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:55:06,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:11,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:16,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:17,261 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:55:17,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:55:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:55:17,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:55:19,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:55:19,795 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:55:19,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:55:19,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:55:20,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:55:21,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:55:21,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:26,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:31,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:32,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:55:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:55:32,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:55:35,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:55:35,660 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:55:35,661 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:55:35,663 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:55:36,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:55:37,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:55:37,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:42,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:47,264 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:55:47,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:55:47,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:55:47,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:55:47,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:51,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:55:51,842 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:55:51,842 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:55:51,843 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:55:52,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:55:52,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:55:53,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:55:57,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:02,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:56:02,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:56:02,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:56:03,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:08,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:56:08,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:56:08,011 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:56:08,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:56:08,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:56:09,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:09,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:56:14,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:17,269 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:56:17,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:56:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:56:17,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:56:19,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:23,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:56:23,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:56:23,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:56:23,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:56:24,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:56:24,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:25,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:56:29,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:32,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:56:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:56:32,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:56:35,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:39,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:56:39,945 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:56:39,945 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:56:39,947 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:56:40,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:56:40,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:41,137 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:56:45,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:47,271 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:56:47,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:56:47,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:56:47,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:56:51,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:56:56,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:56:56,224 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:56:56,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:56:56,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:56:57,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:56:57,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:56:57,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:02,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:02,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:57:02,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:57:02,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:57:07,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:12,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:57:12,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:57:12,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:57:12,304 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:57:13,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:57:13,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:57:13,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:17,274 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:57:17,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:57:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:57:17,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:57:18,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:23,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:28,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:57:28,258 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:57:28,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:57:28,260 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:57:29,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:57:29,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:57:29,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:32,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:57:32,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:57:32,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:57:34,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:39,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:44,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:57:44,448 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:57:44,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:57:44,450 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:57:45,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:57:45,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:57:45,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:47,277 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:57:47,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:57:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:57:47,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:57:50,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:57:55,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:00,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:58:00,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:58:00,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:58:00,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:58:01,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:58:01,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:58:01,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:02,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:58:02,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:58:02,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:58:06,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:11,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:16,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:58:16,546 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:58:16,547 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:58:16,548 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:58:17,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:58:17,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:58:17,280 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:58:17,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:17,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:58:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:58:17,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:58:22,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:27,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:32,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:58:32,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:58:32,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:58:32,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:58:32,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:58:32,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:58:32,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:32,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:58:33,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:58:33,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:58:37,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:42,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:47,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:58:47,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:58:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:58:47,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:58:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:48,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:58:48,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:58:48,850 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:58:48,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:58:49,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:58:49,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:58:53,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:58:58,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:02,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:59:02,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:59:02,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:59:04,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:59:04,700 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:59:04,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:59:04,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:04,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:59:05,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:59:05,245 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:59:09,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:14,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:17,286 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:59:17,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:59:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:59:17,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:59:19,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:20,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:59:20,945 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:59:20,946 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:59:20,947 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:59:21,257 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:59:23,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:59:24,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:29,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:32,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:59:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:59:32,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:59:35,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:36,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:59:37,000 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:59:37,000 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:59:37,002 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:59:37,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:59:39,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:59:41,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:46,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:47,290 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 21:59:47,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 21:59:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 21:59:47,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 21:59:51,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 21:59:53,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 21:59:53,050 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 21:59:53,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 21:59:53,052 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 21:59:53,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 21:59:55,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 21:59:57,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:02,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:02,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:00:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:00:02,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:00:07,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:09,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:00:09,176 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:00:09,176 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:00:09,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:00:09,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:00:11,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:00:13,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:17,292 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:00:17,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:00:17,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:00:17,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:00:18,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:23,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:25,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:00:25,201 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:00:25,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:00:25,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:00:25,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:00:27,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:00:29,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:32,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:00:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:00:32,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:00:34,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:39,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:41,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:00:41,139 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:00:41,139 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:00:41,141 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:00:41,316 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:00:43,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:00:45,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:47,295 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:00:47,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:00:47,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:00:47,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:00:50,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:55,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:00:57,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:00:57,277 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:00:57,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:00:57,280 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:00:57,328 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:00:59,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:01:01,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:02,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:01:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:01:02,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:01:06,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:11,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:13,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:01:13,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:01:13,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:01:13,249 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:01:13,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:01:15,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:01:17,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:17,298 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:01:17,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:01:17,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:01:17,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:01:22,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:27,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:29,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:01:29,417 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:01:29,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:01:29,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:01:30,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:01:31,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:01:32,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:01:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:01:32,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:01:32,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:37,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:42,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:45,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:01:45,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:01:45,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:01:45,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:01:45,365 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:01:47,302 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:01:47,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:01:47,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:01:47,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:01:47,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:01:48,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:53,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:01:58,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:01,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:02:01,316 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:02:01,317 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:02:01,318 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:02:01,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:02:02,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:02:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:02:02,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:02:03,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:02:04,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:09,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:14,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:17,305 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:02:17,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:02:17,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:02:17,358 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:02:17,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:02:17,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:02:17,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:02:17,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:02:17,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:02:19,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:02:19,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:24,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:29,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:32,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:02:32,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:02:32,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:02:33,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:02:33,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:02:33,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:02:33,581 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:02:34,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:02:35,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:02:35,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:40,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:45,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:47,308 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:02:47,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:02:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:02:47,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:02:49,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:02:49,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:02:49,631 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:02:49,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:02:50,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:02:50,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:02:51,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:02:55,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:00,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:02,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:03:02,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:03:02,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:03:05,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:03:05,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:03:05,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:03:05,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:05,671 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:03:06,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:03:07,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:03:10,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:15,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:17,312 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:03:17,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:03:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:03:17,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:03:20,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:21,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:03:21,623 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:03:21,624 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:03:21,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:03:22,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:03:23,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:03:26,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:31,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:32,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:03:32,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:03:32,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:03:36,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:37,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:03:37,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:03:37,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:03:37,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:03:38,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:03:39,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:03:42,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:47,315 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:03:47,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:03:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:03:47,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:03:47,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:52,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:03:53,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:03:53,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:03:53,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:03:53,953 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:03:54,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:03:55,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:03:57,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:02,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:04:02,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:04:02,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:04:03,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:08,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:09,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:04:09,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:04:09,805 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:04:09,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:04:10,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:04:11,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:04:13,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:17,318 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:04:17,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:04:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:04:17,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:04:19,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:24,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:25,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:04:25,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:04:25,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:04:25,871 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:04:26,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:04:27,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:04:29,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:32,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:04:32,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:04:32,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:04:35,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:40,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:42,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:04:42,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:04:42,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:04:42,110 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:04:42,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:04:43,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:04:46,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:47,322 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:04:47,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:04:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:04:47,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:04:51,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:56,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:04:57,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:04:57,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:04:57,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:04:57,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:04:58,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:04:59,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:05:01,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:02,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:05:02,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:05:02,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:05:07,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:12,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:14,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:05:14,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:05:14,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:05:14,011 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:05:14,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:05:15,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:05:17,323 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:05:17,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:05:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:05:17,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:05:17,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:22,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:27,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:30,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:05:30,191 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:05:30,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:05:30,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:05:30,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:05:31,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:05:32,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:05:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:05:32,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:05:33,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:38,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:43,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:46,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:05:46,264 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:05:46,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:05:46,266 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:05:46,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:05:47,327 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:05:47,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:05:47,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:05:47,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:05:47,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:05:49,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:54,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:05:59,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:02,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:06:02,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:06:02,340 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:06:02,340 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:06:02,342 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:06:02,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:06:02,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:06:02,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:06:03,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:06:05,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:10,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:15,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:17,330 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:06:17,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:06:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:06:17,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:06:18,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:06:18,598 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:06:18,598 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:06:18,600 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:06:19,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:06:19,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:06:21,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:26,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:31,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:32,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:06:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:06:32,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:06:34,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:06:34,644 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:06:34,645 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:06:34,646 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:06:35,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:06:35,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:06:36,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:41,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:46,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:47,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:06:47,335 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:06:47,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:06:47,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:06:50,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:06:50,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:06:50,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:06:50,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:06:51,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:06:51,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:06:51,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:06:56,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:01,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:02,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:07:02,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:07:02,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:07:07,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:07:07,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:07:07,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:07:07,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:07,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:07:07,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:07:07,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:07:12,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:17,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:17,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:07:17,339 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:07:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:07:17,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:07:22,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:23,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:07:23,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:07:23,355 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:07:23,357 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:07:23,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:07:23,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:07:28,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:32,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:07:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:07:32,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:07:33,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:38,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:39,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:07:39,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:07:39,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:07:39,231 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:07:39,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:07:39,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:07:44,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:47,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:07:47,343 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:07:47,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:07:47,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:07:49,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:54,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:07:55,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:07:55,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:07:55,484 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:07:55,486 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:07:55,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:07:57,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:08:00,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:02,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:08:02,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:08:02,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:08:05,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:10,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:11,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:08:11,699 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:08:11,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:08:11,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:08:12,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:08:13,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:08:16,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:17,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:08:17,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:08:17,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:08:17,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:08:21,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:26,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:27,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:08:27,746 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:08:27,746 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:08:27,748 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:08:28,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:08:29,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:08:31,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:32,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:08:32,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:08:32,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:08:37,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:42,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:43,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:08:43,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:08:43,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:08:43,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:08:44,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:08:45,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:08:47,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:08:47,394 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:08:47,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:08:47,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:08:48,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:53,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:08:58,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:00,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:09:00,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:09:00,091 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:09:00,092 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:09:00,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:09:01,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:09:02,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:09:02,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:09:02,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:09:03,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:08,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:13,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:16,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:09:16,023 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:09:16,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:09:16,025 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:09:16,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:09:17,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:09:17,397 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:09:17,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:09:17,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:09:17,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:09:19,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:24,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:29,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:32,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:09:32,193 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:09:32,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:09:32,195 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:09:32,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:09:32,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:09:32,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:09:32,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:09:33,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:09:34,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:39,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:44,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:47,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:09:47,400 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:09:47,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:09:47,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:09:48,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:09:48,057 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:09:48,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:09:48,059 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:09:48,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:09:49,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:09:50,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:09:55,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:00,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:02,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:10:02,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:10:02,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:10:04,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:10:04,098 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:10:04,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:10:04,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:10:04,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:10:05,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:05,739 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:10:10,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:15,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:17,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:10:17,403 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:10:17,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:10:17,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:10:19,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:10:19,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:10:19,965 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:10:19,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:10:20,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:10:20,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:21,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:10:25,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:30,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:32,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:10:32,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:10:32,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:10:36,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:10:36,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:10:36,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:10:36,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:36,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:10:36,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:10:37,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:10:41,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:46,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:47,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:10:47,405 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:10:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:10:47,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:10:51,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:10:52,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:10:52,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:10:52,091 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:10:52,093 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:10:52,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:10:53,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:10:57,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:02,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:02,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:11:02,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:11:02,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:11:07,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:08,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:11:08,232 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:11:08,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:11:08,234 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:11:08,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:11:09,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:11:13,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:17,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:11:17,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:11:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:11:17,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:11:18,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:23,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:24,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:11:24,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:11:24,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:11:24,290 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:11:24,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:11:25,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:11:29,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:32,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:11:32,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:11:32,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:11:34,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:39,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:40,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:11:40,431 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:11:40,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:11:40,433 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:11:40,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:11:41,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:11:45,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:47,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:11:47,414 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:11:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:11:47,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:11:50,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:55,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:11:56,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:11:56,478 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:11:56,479 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:11:56,481 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:11:56,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:11:57,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:12:01,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:02,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:12:02,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:12:02,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:12:06,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:11,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:12,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:12:12,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:12:12,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:12:12,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:12:12,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:12:13,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:12:17,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:17,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:12:17,416 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:12:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:12:17,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:12:22,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:27,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:28,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:12:28,877 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:12:28,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:12:28,879 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:12:29,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:12:30,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:12:32,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:12:32,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:12:32,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:12:33,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:38,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:43,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:44,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:12:44,741 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:12:44,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:12:44,743 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:12:44,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:12:46,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:12:47,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:12:47,419 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:12:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:12:47,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:12:48,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:53,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:12:58,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:00,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:13:00,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:13:00,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:13:00,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:13:00,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:13:02,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:13:02,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:13:02,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:13:02,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:13:04,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:09,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:14,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:16,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:13:16,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:13:16,862 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:13:16,863 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:13:16,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:13:17,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:13:17,422 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:13:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:13:17,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:13:18,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:13:20,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:25,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:30,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:32,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:13:32,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:13:32,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:13:32,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:13:32,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:13:32,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:13:32,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:13:32,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:13:34,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:13:35,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:40,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:45,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:47,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:13:47,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:13:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:13:47,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:13:48,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:13:48,787 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:13:48,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:13:48,789 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:13:48,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:13:50,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:13:50,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:13:55,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:00,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:02,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:14:02,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:14:02,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:14:04,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:14:04,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:14:04,949 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:14:04,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:14:05,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:14:05,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:06,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:14:10,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:15,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:17,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:14:17,429 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:14:17,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:14:17,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:14:20,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:14:20,887 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:14:20,888 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:14:20,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:14:20,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:14:21,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:22,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:14:26,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:31,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:32,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:14:32,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:14:32,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:14:36,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:14:36,939 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:14:36,939 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:14:36,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:36,941 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:14:36,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:14:38,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:14:41,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:46,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:47,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:14:47,432 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:14:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:14:47,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:14:52,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:14:53,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:14:53,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:14:53,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:14:53,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:14:53,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:14:54,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:14:58,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:02,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:15:02,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:15:02,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:15:03,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:08,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:09,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:15:09,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:15:09,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:15:09,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:15:09,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:15:10,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:15:14,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:17,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:15:17,439 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:15:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:15:17,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:15:19,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:24,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:25,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:15:25,377 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:15:25,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:15:25,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:15:25,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:15:26,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:15:30,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:32,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:15:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:15:32,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:15:35,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:40,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:41,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:15:41,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:15:41,677 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:15:41,679 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:15:41,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:15:42,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:15:46,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:47,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:15:47,441 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:15:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:15:47,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:15:52,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:57,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:15:57,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:15:57,928 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:15:57,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:15:57,930 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:15:58,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:15:59,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:16:02,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:16:02,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:16:02,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:16:02,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:07,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:12,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:13,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:16:13,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:16:13,797 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:16:13,799 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:16:14,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:16:15,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:16:17,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:16:17,445 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:16:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:16:17,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:16:18,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:23,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:28,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:30,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:16:30,064 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:16:30,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:16:30,066 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:16:31,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:16:32,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:16:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:16:32,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:16:33,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:16:33,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:38,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:43,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:46,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:16:46,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:16:46,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:16:46,290 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:16:47,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:16:47,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:16:47,448 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:16:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:16:47,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:16:49,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:16:49,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:54,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:16:59,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:02,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:17:02,332 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:17:02,333 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:17:02,334 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:17:02,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:17:02,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:17:02,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:17:03,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:17:05,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:17:05,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:10,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:15,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:17,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:17:17,452 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:17:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:17:17,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:17:18,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:17:18,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:17:18,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:17:18,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:17:19,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:17:21,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:17:21,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:26,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:31,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:32,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:17:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:17:32,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:17:34,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:17:34,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:17:34,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:17:34,847 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:17:35,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:17:36,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:37,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:17:41,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:46,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:47,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:17:47,455 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:17:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:17:47,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:17:50,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:17:50,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:17:50,776 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:17:50,777 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:17:51,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:17:52,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:17:53,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:17:57,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:02,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:18:02,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:18:02,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:18:03,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:07,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:18:07,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:18:07,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:18:07,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:18:07,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:18:09,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:09,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:18:14,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:17,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:18:17,459 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:18:17,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:18:17,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:18:19,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:22,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:18:22,897 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:18:22,897 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:18:22,900 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:18:23,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:18:24,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:25,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:18:29,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:32,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:18:32,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:18:32,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:18:35,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:39,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:18:39,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:18:39,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:18:39,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:18:39,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:18:41,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:41,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:18:46,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:47,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:18:47,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:18:47,463 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:18:47,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:18:51,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:54,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:18:54,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:18:54,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:18:54,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:18:55,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:18:56,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:18:57,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:19:01,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:02,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:19:02,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:19:02,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:19:07,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:11,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:19:11,050 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:19:11,050 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:19:11,052 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:19:11,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:19:13,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:13,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:19:17,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:19:17,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:19:17,467 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:19:17,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:19:18,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:23,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:27,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:19:27,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:19:27,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:19:27,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:19:27,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:19:29,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:29,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:19:32,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:19:32,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:19:32,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:19:34,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:39,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:43,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:19:43,171 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:19:43,172 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:19:43,173 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:19:44,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:19:45,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:45,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:19:47,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:19:47,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:19:47,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:19:47,696 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:19:50,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:55,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:19:59,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:19:59,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:19:59,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:19:59,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:20:00,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:20:01,188 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:20:01,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:02,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:20:02,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:20:02,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:20:06,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:11,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:15,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:20:15,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:20:15,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:20:15,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:20:16,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:20:17,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:20:17,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:17,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:20:17,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:20:17,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:20:17,703 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:20:22,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:27,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:31,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:20:31,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:20:31,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:20:31,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:20:32,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:20:32,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:20:32,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:20:32,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:20:32,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:33,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:20:37,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:42,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:47,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:20:47,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:20:47,402 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:20:47,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:20:47,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:20:47,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:20:47,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:20:47,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:20:48,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:20:48,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:49,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:20:53,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:20:58,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:02,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:21:02,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:21:02,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:21:03,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:03,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:21:03,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:21:03,748 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:21:03,749 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:21:04,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:21:05,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:21:08,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:13,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:17,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:21:17,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:21:17,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:21:17,657 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:21:19,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:21:19,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:21:19,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:21:19,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:19,613 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:21:20,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:21:21,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:21:24,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:29,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:32,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:21:32,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:21:32,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:21:34,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:35,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:21:35,570 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:21:35,571 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:21:35,572 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:21:36,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:21:37,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:21:40,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:45,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:47,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:21:47,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:21:47,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:21:47,694 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:21:50,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:21:51,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:21:51,852 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:21:51,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:21:51,855 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:21:52,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:21:53,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:21:55,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:00,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:02,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:22:02,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:22:02,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:22:06,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:07,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:22:07,711 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:22:07,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:22:07,714 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:22:08,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:22:09,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:22:11,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:16,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:17,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:22:17,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:22:17,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:22:17,708 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:22:22,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:23,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:22:23,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:22:23,678 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:22:23,680 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:22:24,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:22:25,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:22:28,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:32,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:22:32,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:22:32,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:22:34,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:39,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:39,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:22:39,926 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:22:39,926 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:22:39,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:22:40,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:22:41,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:22:44,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:47,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:22:47,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:22:47,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:22:47,680 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:22:50,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:55,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:22:56,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:22:56,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:22:56,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:22:56,031 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:22:56,315 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:22:57,315 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:23:01,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:02,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:23:02,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:23:02,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:23:06,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:11,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:12,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:23:12,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:23:12,007 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:23:12,008 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:23:12,326 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:23:13,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:23:17,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:17,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:23:17,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:23:17,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:23:17,720 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:23:22,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:27,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:28,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:23:28,360 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:23:28,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:23:28,363 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:23:29,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:23:29,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:23:32,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:23:32,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:23:32,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:23:32,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:37,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:42,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:44,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:23:44,310 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:23:44,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:23:44,312 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:23:44,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:23:45,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:23:47,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:23:47,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:23:47,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:23:47,737 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:23:48,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:53,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:23:58,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:00,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:24:00,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:24:00,477 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:24:00,479 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:24:01,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:24:01,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:24:02,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:24:02,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:24:02,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:24:04,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:09,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:14,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:16,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:24:16,651 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:24:16,651 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:24:16,653 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:24:17,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:24:17,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:24:17,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:24:17,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:24:17,721 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:24:19,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:24:19,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:24,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:29,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:32,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:24:32,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:24:32,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:24:32,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:24:32,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:24:32,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:24:32,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:24:33,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:24:34,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:35,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:24:39,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:44,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:47,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:24:47,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:24:47,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:24:47,675 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:24:48,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:24:48,771 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:24:48,771 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:24:48,773 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:24:49,395 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:24:50,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:24:51,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:24:55,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:00,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:02,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:25:02,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:25:02,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:25:04,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:25:04,945 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:25:04,945 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:25:04,948 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:25:05,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:25:05,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:07,408 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:10,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:15,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:17,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:25:17,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:25:17,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:25:17,721 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:25:20,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:25:21,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:25:21,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:25:21,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:21,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:25:21,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:25:23,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:26,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:27,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:31,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:31,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:32,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:25:33,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:25:33,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:25:33,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:37,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:39,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:41,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:42,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:43,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:25:45,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:25:47,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:25:47,563 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:25:47,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:25:47,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:25:48,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:53,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:58,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:25:59,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:25:59,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:25:59,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:25:59,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:26:00,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:26:01,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:26:02,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:26:02,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:26:02,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:26:03,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:08,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:13,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:16,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:26:16,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:26:16,145 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:26:16,148 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:26:16,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:26:17,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:26:17,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:26:17,565 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:26:17,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:26:17,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:26:19,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:24,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:29,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:32,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:26:32,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:26:32,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:26:32,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:26:32,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:26:32,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:26:32,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:26:32,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:26:33,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:26:35,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:40,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:45,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:47,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:26:47,568 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:26:47,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:26:47,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:26:48,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:26:48,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:26:48,534 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:26:48,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:26:49,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:26:49,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:26:50,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:26:55,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:00,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:02,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:27:02,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:27:02,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:27:04,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:27:04,395 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:27:04,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:27:04,396 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:27:04,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:27:05,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:27:06,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:11,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:16,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:17,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:27:17,572 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:27:17,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:27:17,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:27:20,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:27:20,690 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:27:20,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:27:20,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:27:21,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:27:21,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:27:21,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:26,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:31,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:32,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:27:32,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:27:32,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:27:36,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:27:36,651 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:27:36,652 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:27:36,653 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:27:37,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:27:37,521 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:27:37,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:42,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:47,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:27:47,575 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:27:47,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:27:47,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:27:48,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:52,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:27:52,829 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:27:52,830 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:27:52,831 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:27:53,532 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:27:53,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:27:55,533 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:27:58,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:02,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:28:02,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:28:02,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:28:04,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:08,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:28:08,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:28:08,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:28:08,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:28:09,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:28:09,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:28:09,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:14,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:17,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:28:17,578 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:28:17,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:28:17,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:28:20,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:24,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:28:24,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:28:24,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:28:24,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:28:25,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:28:25,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:27,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:28:30,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:32,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:28:32,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:28:32,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:28:36,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:40,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:28:40,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:28:40,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:28:40,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:28:41,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:28:41,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:28:41,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:46,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:47,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:28:47,583 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:28:47,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:28:47,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:28:52,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:57,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:28:57,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:28:57,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:28:57,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:28:57,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:28:58,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:28:59,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:29:02,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:29:02,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:29:02,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:29:03,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:08,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:12,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:29:12,973 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:29:12,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:29:12,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:29:13,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:29:13,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:15,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:29:17,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:29:17,586 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:29:17,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:29:17,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:29:19,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:24,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:29,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:29:29,050 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:29:29,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:29:29,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:29:29,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:29:30,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:31,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:29:32,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:29:32,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:29:32,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:29:35,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:40,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:45,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:45,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:29:45,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:29:45,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:29:45,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:29:45,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:29:47,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:29:47,590 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:29:47,618 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:29:47,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:29:47,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:29:51,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:29:56,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:01,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:01,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:30:01,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:30:01,252 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:30:01,253 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:30:01,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:30:02,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:30:02,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:30:02,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:30:03,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:30:06,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:11,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:16,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:17,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:30:17,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:30:17,396 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:30:17,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:30:17,398 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:30:17,593 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:30:17,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:30:17,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:30:17,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:30:19,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:30:21,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:26,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:31,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:32,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:30:32,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:30:32,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:30:33,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:30:33,257 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:30:33,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:30:33,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:30:33,653 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:30:35,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:30:36,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:41,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:46,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:47,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:30:47,597 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:30:47,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:30:47,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:30:49,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:30:49,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:30:49,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:30:49,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:30:49,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:30:51,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:30:51,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:30:56,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:01,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:02,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:31:02,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:31:02,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:31:05,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:31:05,148 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:31:05,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:31:05,150 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:31:05,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:31:07,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:07,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:31:12,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:17,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:17,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:31:17,600 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:31:17,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:31:17,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:31:21,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:31:21,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:31:21,234 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:31:21,236 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:31:21,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:31:22,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:23,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:31:27,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:32,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:32,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:31:32,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:31:32,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:31:37,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:31:37,176 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:31:37,176 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:31:37,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:31:37,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:31:38,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:39,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:31:43,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:47,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:31:47,603 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:31:47,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:31:47,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:31:49,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:53,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:31:53,114 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:31:53,115 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:31:53,116 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:31:53,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:31:54,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:31:55,713 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:31:59,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:02,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:32:02,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:32:02,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:32:05,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:09,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:32:09,072 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:32:09,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:32:09,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:32:09,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:32:09,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:32:10,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:15,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:17,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:32:17,608 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:32:17,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:32:17,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:32:20,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:25,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:25,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:32:25,336 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:32:25,336 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:32:25,338 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:32:25,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:32:27,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:32:30,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:32,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:32:32,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:32:32,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:32:36,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:41,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:41,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:32:41,286 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:32:41,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:32:41,288 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:32:41,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:32:43,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:32:46,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:47,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:32:47,609 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:32:47,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:32:47,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:32:52,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:57,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:32:57,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:32:57,224 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:32:57,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:32:57,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:32:57,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:32:59,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:33:02,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:02,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:33:02,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:33:02,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:33:08,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:13,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:13,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:33:13,171 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:33:13,171 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:33:13,173 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:33:13,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:33:15,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:33:17,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:33:17,613 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:33:17,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:33:17,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:33:19,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:24,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:29,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:29,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:33:29,209 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:33:29,209 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:33:29,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:33:29,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:33:31,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:33:32,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:33:32,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:33:32,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:33:34,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:39,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:44,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:45,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:33:45,339 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:33:45,339 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:33:45,341 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:33:45,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:33:47,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:33:47,617 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:33:47,795 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:33:47,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:33:47,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:33:50,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:33:55,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:00,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:01,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:34:01,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:34:01,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:34:01,402 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:34:01,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:34:02,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:34:02,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:34:02,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:34:03,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:34:05,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:10,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:15,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:17,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:34:17,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:34:17,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:34:17,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:34:17,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:34:17,617 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:34:17,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:34:17,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:34:17,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:34:19,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:34:20,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:25,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:30,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:32,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:34:32,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:34:32,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:34:33,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:34:33,526 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:34:33,527 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:34:33,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:34:33,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:34:35,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:35,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:34:40,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:45,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:47,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:34:47,621 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:34:47,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:34:47,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:34:49,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:34:49,371 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:34:49,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:34:49,374 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:34:49,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:34:51,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:34:51,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:34:56,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:01,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:02,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:35:02,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:35:02,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:35:05,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:35:05,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:35:05,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:35:05,644 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:35:05,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:35:06,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:07,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:35:11,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:16,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:17,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:35:17,626 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:35:17,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:35:17,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:35:21,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:35:21,742 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:35:21,742 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:35:21,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:21,744 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:35:21,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:35:23,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:35:26,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:31,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:32,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:35:32,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:35:32,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:35:37,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:37,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:35:37,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:35:37,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:35:37,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:35:37,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:35:39,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:35:42,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:47,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:35:47,631 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:35:47,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:47,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:35:47,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:35:53,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:35:53,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:35:53,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:35:53,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:35:53,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:35:54,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:35:55,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:35:58,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:02,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:36:02,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:36:02,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:36:04,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:09,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:09,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:36:09,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:36:09,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:36:09,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:36:09,899 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:36:11,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:36:14,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:17,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:36:17,633 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:36:17,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:36:17,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:36:20,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:25,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:26,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:36:26,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:36:26,323 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:36:26,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:36:26,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:36:27,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:36:30,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:32,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:36:32,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:36:32,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:36:36,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:41,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:42,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:36:42,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:36:42,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:36:42,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:36:42,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:36:43,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:36:46,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:47,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:36:47,636 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:36:47,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:36:47,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:36:52,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:57,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:36:58,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:36:58,533 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:36:58,533 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:36:58,535 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:36:58,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:36:59,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:37:02,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:37:02,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:02,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:37:02,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:37:08,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:13,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:14,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:37:14,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:37:14,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:37:14,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:37:14,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:37:15,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:37:17,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:37:17,642 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:37:17,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:37:17,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:37:18,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:23,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:28,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:30,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:37:30,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:37:30,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:37:30,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:37:30,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:37:31,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:37:32,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:37:32,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:37:32,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:37:33,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:38,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:43,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:46,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:37:46,844 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:37:46,844 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:37:46,846 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:37:46,972 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:37:47,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:37:47,643 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:37:47,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:37:47,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:37:47,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:37:49,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:54,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:37:59,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:02,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:38:02,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:38:02,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:38:03,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:38:03,112 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:38:03,112 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:38:03,113 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:38:03,985 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:38:03,985 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:38:04,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:09,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:14,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:17,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:38:17,646 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:38:17,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:38:17,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:38:19,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:38:19,045 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:38:19,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:38:19,047 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:38:19,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:38:19,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:38:20,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:25,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:30,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:32,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:38:32,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:38:32,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:38:35,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:35,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:38:35,281 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:38:35,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:38:35,284 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:38:36,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:38:36,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:38:40,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:45,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:47,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:38:47,649 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:38:47,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:38:47,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:38:51,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:38:51,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:38:51,246 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:38:51,246 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:38:51,248 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:38:52,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:38:52,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:38:56,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:01,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:02,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:39:02,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:39:02,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:39:07,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:07,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:39:07,404 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:39:07,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:39:07,406 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:39:08,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:39:08,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:39:12,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:17,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:39:17,653 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:39:17,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:17,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:39:17,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:39:23,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:23,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:39:23,280 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:39:23,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:39:23,282 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:39:24,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:39:24,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:39:28,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:32,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:39:32,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:39:32,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:39:34,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:39,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:39,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:39:39,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:39:39,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:39:39,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:39:40,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:39:40,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:39:44,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:47,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:39:47,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:39:47,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:39:47,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:39:50,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:55,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:39:55,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:39:55,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:39:55,466 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:39:55,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:39:56,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:39:56,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:40:00,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:02,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:40:02,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:40:02,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:40:06,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:11,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:11,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:40:11,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:40:11,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:40:11,396 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:40:12,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:40:12,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:40:16,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:17,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:40:17,659 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:40:17,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:40:17,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:40:22,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:27,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:27,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:40:27,451 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:40:27,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:40:27,453 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:40:28,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:40:28,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:40:32,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:40:32,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:32,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:40:32,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:40:38,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:43,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:43,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:40:43,307 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:40:43,307 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:40:43,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:40:44,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:40:44,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:40:47,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:40:47,663 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:40:47,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:40:47,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:40:48,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:53,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:58,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:40:59,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:40:59,580 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:40:59,581 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:40:59,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:41:00,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:41:00,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:41:02,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:41:02,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:41:02,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:41:04,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:09,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:14,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:15,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:41:15,426 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:41:15,427 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:41:15,428 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:41:16,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:41:16,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:41:17,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:41:17,665 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:41:17,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:41:17,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:41:20,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:25,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:30,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:31,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:41:31,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:41:31,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:41:31,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:41:32,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:41:32,137 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:41:32,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:41:32,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:41:32,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:41:35,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:40,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:45,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:47,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:41:47,347 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:41:47,347 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:41:47,349 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:41:47,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:41:47,669 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:41:47,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:41:47,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:41:48,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:41:48,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:41:50,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:41:55,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:00,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:02,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:42:02,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:42:02,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:42:03,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:42:03,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:42:03,312 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:42:03,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:42:04,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:42:04,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:42:05,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:10,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:15,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:17,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:42:17,672 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:42:17,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:42:17,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:42:19,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:42:19,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:42:19,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:42:19,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:42:20,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:42:20,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:42:20,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:25,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:30,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:32,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:42:32,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:42:32,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:42:35,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:42:35,657 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:42:35,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:42:35,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:35,659 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:42:36,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:42:36,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:42:40,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:45,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:47,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:42:47,676 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:42:47,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:42:47,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:42:51,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:42:51,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:42:51,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:42:51,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:42:51,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:42:52,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:42:52,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:42:56,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:01,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:02,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:43:02,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:43:02,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:43:07,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:07,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:43:07,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:43:07,578 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:43:07,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:43:08,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:43:08,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:43:12,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:17,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:43:17,678 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:43:17,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:17,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:43:17,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:43:23,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:23,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:43:23,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:43:23,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:43:23,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:43:24,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:43:24,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:43:28,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:32,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:43:32,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:43:32,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:43:34,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:39,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:39,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:43:39,570 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:43:39,570 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:43:39,572 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:43:40,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:43:40,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:43:44,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:47,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:43:47,684 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:43:47,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:43:47,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:43:50,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:55,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:43:55,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:43:55,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:43:55,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:43:55,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:43:56,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:43:56,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:44:00,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:02,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:44:02,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:44:02,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:44:06,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:11,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:11,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:44:11,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:44:11,797 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:44:11,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:44:12,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:44:12,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:44:16,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:17,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:44:17,689 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:44:17,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:44:17,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:44:22,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:27,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:27,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:44:27,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:44:27,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:44:27,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:44:28,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:44:30,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:44:32,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:32,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:44:32,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:44:32,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:44:38,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:43,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:43,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:44:43,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:44:43,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:44:43,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:44:44,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:44:46,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:44:47,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:44:47,690 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:44:47,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:44:47,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:44:49,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:54,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:59,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:44:59,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:44:59,851 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:44:59,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:44:59,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:45:00,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:45:00,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:45:02,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:45:02,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:45:02,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:45:04,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:09,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:14,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:16,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:45:16,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:45:16,113 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:45:16,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:45:16,296 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:45:17,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:45:17,693 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:45:17,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:45:17,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:45:18,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:45:19,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:24,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:29,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:31,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:45:31,967 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:45:31,967 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:45:31,969 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:45:32,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:45:32,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:45:32,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:45:32,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:45:32,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:45:35,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:40,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:45,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:47,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:45:47,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:45:47,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:45:47,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:45:47,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:45:48,151 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:45:48,152 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:45:48,153 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:45:48,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:45:50,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:45:50,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:45:55,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:00,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:02,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:46:02,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:46:02,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:46:04,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:46:04,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:46:04,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:46:04,011 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:46:04,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:46:06,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:06,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:46:11,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:16,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:17,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:46:17,700 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:46:17,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:46:17,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:46:20,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:46:20,059 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:46:20,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:46:20,060 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:46:20,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:46:21,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:22,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:46:26,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:31,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:32,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:46:32,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:46:32,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:46:36,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:36,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:46:36,104 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:46:36,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:46:36,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:46:36,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:46:38,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:46:41,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:46,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:47,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:46:47,703 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:46:47,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:46:47,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:46:51,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:46:52,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:46:52,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:46:52,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:46:52,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:46:52,365 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:46:54,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:46:56,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:01,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:02,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:47:02,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:47:02,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:47:07,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:08,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:47:08,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:47:08,234 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:47:08,236 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:47:08,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:47:10,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:47:12,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:17,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:17,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:47:17,707 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:47:17,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:47:17,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:47:23,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:24,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:47:24,312 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:47:24,312 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:47:24,314 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:47:24,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:47:26,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:47:28,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:32,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:47:32,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:47:32,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:47:34,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:39,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:40,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:47:40,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:47:40,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:47:40,668 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:47:41,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:47:42,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:47:44,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:47,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:47:47,710 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:47:47,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:47:47,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:47:50,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:55,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:47:56,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:47:56,615 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:47:56,615 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:47:56,617 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:47:57,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:47:58,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:48:00,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:02,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:48:02,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:48:02,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:48:06,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:11,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:12,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:48:12,868 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:48:12,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:48:12,870 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:48:13,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:48:14,424 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:48:16,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:17,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:48:17,715 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:48:17,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:48:17,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:48:22,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:27,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:28,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:48:28,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:48:28,883 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:48:28,884 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:48:29,435 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:48:30,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:48:32,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:32,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:48:32,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:48:32,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:48:38,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:43,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:45,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:48:45,156 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:48:45,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:48:45,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:48:45,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:48:46,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:48:47,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:48:47,718 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:48:47,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:48:47,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:48:48,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:53,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:48:58,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:01,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:49:01,213 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:49:01,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:49:01,216 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:49:01,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:49:02,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:49:02,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:49:02,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:49:02,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:49:04,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:09,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:14,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:17,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:49:17,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:49:17,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:49:17,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:49:17,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:49:17,470 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:49:17,723 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:49:17,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:49:17,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:49:18,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:49:19,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:24,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:29,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:32,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:49:32,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:49:32,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:49:33,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:49:33,424 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:49:33,424 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:49:33,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:49:33,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:49:34,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:34,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:49:39,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:44,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:47,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:49:47,725 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:49:47,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:49:47,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:49:49,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:49:49,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:49:49,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:49:49,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:49,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:49:49,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:49:50,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:49:54,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:49:59,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:02,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:50:02,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:50:02,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:50:05,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:05,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:50:05,786 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:50:05,786 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:50:05,788 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:50:06,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:50:06,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:50:10,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:15,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:17,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:50:17,728 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:50:17,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:50:17,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:50:21,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:21,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:50:21,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:50:21,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:50:21,750 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:50:22,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:50:22,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:50:26,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:31,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:32,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:50:32,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:50:32,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:50:37,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:37,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:50:37,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:50:37,789 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:50:37,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:50:38,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:50:38,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:50:42,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:47,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:50:47,731 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:50:47,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:50:47,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:50:48,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:53,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:50:53,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:50:53,736 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:50:53,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:50:53,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:50:54,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:50:54,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:50:58,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:02,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:51:02,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:51:02,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:51:04,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:09,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:09,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:51:09,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:51:09,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:51:09,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:51:10,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:51:10,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:51:14,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:17,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:51:17,733 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:51:17,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:51:17,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:51:20,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:25,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:25,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:51:25,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:51:25,834 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:51:25,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:51:26,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:51:26,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:51:30,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:32,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:51:32,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:51:32,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:51:36,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:41,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:41,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:51:41,793 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:51:41,794 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:51:41,795 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:51:42,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:51:42,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:51:46,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:47,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:51:47,737 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:51:47,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:51:47,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:51:52,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:57,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:51:57,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:51:57,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:51:57,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:51:57,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:51:58,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:51:58,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:52:02,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:02,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:52:02,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:52:02,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:52:08,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:13,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:13,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:52:13,885 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:52:13,885 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:52:13,887 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:52:14,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:52:14,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:52:17,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:52:17,740 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:52:17,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:52:17,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:52:19,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:24,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:29,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:30,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:52:30,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:52:30,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:52:30,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:52:30,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:52:30,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:52:32,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:52:32,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:52:32,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:52:35,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:40,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:45,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:46,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:52:46,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:52:46,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:52:46,231 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:52:46,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:52:46,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:52:47,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:52:47,745 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:52:47,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:52:47,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:52:51,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:52:56,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:01,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:02,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:53:02,263 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:53:02,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:53:02,265 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:53:02,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:53:02,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:53:02,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:53:02,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:53:02,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:53:07,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:12,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:17,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:17,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:53:17,749 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:53:17,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:53:17,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:53:18,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:53:18,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:53:18,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:53:18,308 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:53:18,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:53:19,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:53:22,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:27,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:32,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:32,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:53:32,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:53:32,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:53:34,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:53:34,400 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:53:34,400 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:53:34,402 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:53:34,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:53:35,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:53:37,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:42,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:47,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:53:47,753 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:53:47,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:47,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:53:47,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:53:50,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:53:50,568 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:53:50,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:53:50,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:53:50,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:53:53,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:53:53,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:53:58,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:02,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:54:02,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:54:02,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:54:04,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:06,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:54:06,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:54:06,610 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:54:06,612 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:54:06,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:54:09,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:09,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:54:14,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:17,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:54:17,757 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:54:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:54:17,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:54:20,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:22,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:54:22,688 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:54:22,688 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:54:22,690 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:54:22,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:54:25,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:25,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:54:30,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:32,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:54:32,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:54:32,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:54:36,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:38,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:54:38,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:54:38,805 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:54:38,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:54:39,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:54:41,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:54:41,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:46,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:47,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:54:47,759 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:54:47,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:54:47,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:54:52,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:54:54,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:54:54,851 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:54:54,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:54:54,853 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:54:55,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:54:57,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:54:57,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:02,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:55:02,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:55:02,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:55:03,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:08,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:10,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:55:10,704 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:55:10,704 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:55:10,706 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:55:10,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:55:13,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:13,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:55:17,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:55:17,762 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:55:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:55:17,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:55:19,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:24,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:26,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:55:26,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:55:26,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:55:26,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:55:27,739 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:55:29,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:55:29,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:32,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:55:32,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:55:32,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:55:35,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:40,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:42,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:55:42,814 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:55:42,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:55:42,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:55:43,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:55:45,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:55:45,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:47,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:55:47,766 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:55:47,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:55:47,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:55:51,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:56,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:55:58,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:55:58,888 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:55:58,888 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:55:58,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:55:59,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:56:01,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:56:01,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:02,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:56:02,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:56:02,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:56:07,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:12,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:14,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:56:14,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:56:14,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:56:14,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:56:15,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:56:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:56:17,769 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:56:17,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:56:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:56:17,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:56:23,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:28,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:30,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:56:30,918 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:56:30,919 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:56:30,921 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:56:31,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:56:32,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:56:32,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:56:32,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:56:33,786 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:56:34,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:39,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:44,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:47,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:56:47,068 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:56:47,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:56:47,070 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:56:47,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:56:47,772 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:56:47,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:56:47,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:56:47,879 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:56:49,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:56:50,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:56:55,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:00,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:02,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:57:02,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:57:02,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:57:03,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:57:03,153 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:57:03,154 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:57:03,155 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:57:03,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:57:05,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:05,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:57:10,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:15,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:57:17,776 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:57:17,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:57:17,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:57:19,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:57:19,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:57:19,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:57:19,109 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:57:19,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:57:21,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:21,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:57:26,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:31,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:32,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:57:32,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:57:32,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:57:35,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:57:35,282 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:57:35,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:57:35,284 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:57:35,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:57:36,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:37,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:57:41,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:46,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:47,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:57:47,781 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:57:47,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:57:47,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:57:51,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:57:51,431 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:57:51,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:57:51,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:57:51,433 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:57:51,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:57:53,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:57:56,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:01,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:02,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:58:02,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:58:02,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:58:07,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:07,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:58:07,513 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:58:07,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:58:07,515 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:58:07,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:58:09,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:58:12,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:17,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:58:17,782 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:58:17,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:17,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:58:17,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:58:23,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:23,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:58:23,689 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:58:23,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:58:23,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:58:23,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:58:25,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:58:28,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:32,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:58:32,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:58:32,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:58:34,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:39,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:39,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:58:39,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:58:39,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:58:39,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:58:40,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:58:41,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:58:44,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:47,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:58:47,834 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:58:47,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:58:47,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:58:50,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:55,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:58:56,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:58:56,072 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:58:56,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:58:56,074 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:58:56,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:58:57,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:59:01,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:02,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:59:02,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:59:02,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:59:06,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:11,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:12,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:59:12,058 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:59:12,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:59:12,060 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:59:12,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:59:13,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:59:17,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:17,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:59:17,837 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:59:17,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:59:17,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:59:22,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:27,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:28,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:59:28,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:59:28,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:59:28,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:59:28,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:59:29,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:59:32,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:32,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:59:32,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:59:32,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:59:38,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:43,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:44,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 22:59:44,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 22:59:44,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 22:59:44,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 22:59:44,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 22:59:45,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 22:59:47,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 22:59:47,841 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 22:59:47,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 22:59:47,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 22:59:48,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:53,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 22:59:58,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:00,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:00:00,402 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:00:00,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:00:00,404 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:00:00,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:00:01,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:00:02,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:00:02,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:00:02,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:00:04,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:09,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:14,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:16,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:00:16,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:00:16,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:00:16,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:00:16,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:00:17,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:00:17,844 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:00:17,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:00:17,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:00:17,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:00:19,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:24,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:29,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:32,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:00:32,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:00:32,716 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:00:32,717 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:00:32,718 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:00:32,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:00:32,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:00:32,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:00:33,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:00:35,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:40,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:45,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:47,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:00:47,847 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:00:47,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:00:47,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:00:48,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:00:48,836 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:00:48,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:00:48,838 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:00:48,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:00:49,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:00:50,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:00:55,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:00,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:02,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:01:02,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:01:02,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:01:04,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:01:04,893 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:01:04,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:01:04,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:01:04,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:01:05,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:05,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:01:10,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:15,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:17,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:01:17,851 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:01:17,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:01:17,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:01:20,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:01:20,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:01:20,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:01:20,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:20,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:01:21,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:01:22,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:01:25,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:30,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:32,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:01:32,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:01:32,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:01:36,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:36,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:01:36,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:01:36,989 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:01:36,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:01:37,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:01:38,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:01:41,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:46,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:47,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:01:47,861 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:01:47,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:01:47,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:01:52,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:01:52,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:01:52,932 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:01:52,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:01:52,935 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:01:53,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:01:54,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:01:57,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:02,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:02:02,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:02:02,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:02:03,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:08,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:08,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:02:08,981 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:02:08,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:02:08,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:02:09,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:02:10,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:02:13,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:17,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:02:17,864 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:02:17,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:02:17,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:02:19,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:24,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:24,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:02:24,913 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:02:24,913 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:02:24,915 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:02:25,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:02:26,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:02:29,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:32,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:02:32,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:02:32,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:02:35,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:40,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:41,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:02:41,192 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:02:41,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:02:41,194 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:02:42,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:02:44,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:02:45,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:47,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:02:47,867 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:02:47,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:02:47,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:02:50,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:55,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:02:57,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:02:57,363 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:02:57,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:02:57,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:02:58,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:03:00,072 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:03:00,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:02,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:03:02,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:03:02,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:03:06,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:11,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:13,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:03:13,389 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:03:13,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:03:13,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:03:14,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:03:16,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:03:16,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:17,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:03:17,871 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:03:17,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:03:17,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:03:22,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:27,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:29,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:03:29,530 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:03:29,530 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:03:29,531 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:03:30,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:03:32,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:03:32,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:32,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:03:32,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:03:32,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:03:38,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:43,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:45,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:03:45,416 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:03:45,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:03:45,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:03:46,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:03:47,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:03:47,874 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:03:47,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:03:47,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:03:48,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:03:49,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:54,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:03:59,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:01,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:04:01,742 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:04:01,742 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:04:01,744 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:04:02,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:04:02,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:04:02,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:04:02,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:04:04,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:04:04,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:09,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:14,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:17,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:04:17,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:04:17,802 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:04:17,803 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:04:17,804 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:04:17,878 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:04:17,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:04:17,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:04:18,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:04:19,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:20,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:04:24,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:29,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:32,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:04:32,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:04:32,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:04:33,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:04:33,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:04:33,797 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:04:33,799 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:04:34,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:04:34,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:36,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:04:39,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:44,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:47,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:04:47,882 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:04:47,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:04:47,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:04:49,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:04:49,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:04:49,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:04:49,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:49,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:04:50,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:04:52,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:04:54,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:04:59,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:02,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:05:02,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:05:02,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:05:05,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:05,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:05:05,903 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:05:05,903 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:05:05,904 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:05:06,164 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:05:08,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:05:10,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:15,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:17,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:05:17,885 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:05:17,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:05:17,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:05:21,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:21,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:05:21,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:05:21,744 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:05:21,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:05:22,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:05:24,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:05:26,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:31,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:32,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:05:32,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:05:32,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:05:37,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:37,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:05:37,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:05:37,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:05:37,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:05:38,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:05:40,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:05:42,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:47,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:05:47,888 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:05:47,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:05:47,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:05:48,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:53,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:05:53,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:05:53,832 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:05:53,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:05:53,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:05:54,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:05:56,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:05:58,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:02,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:06:02,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:06:02,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:06:04,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:09,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:10,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:06:10,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:06:10,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:06:10,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:06:10,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:06:12,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:06:15,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:17,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:06:17,892 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:06:17,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:06:17,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:06:20,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:25,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:26,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:06:26,088 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:06:26,088 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:06:26,090 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:06:26,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:06:28,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:06:31,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:32,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:06:32,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:06:32,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:06:36,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:41,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:42,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:06:42,161 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:06:42,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:06:42,163 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:06:42,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:06:44,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:06:47,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:47,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:06:47,901 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:06:47,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:06:47,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:06:52,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:57,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:06:58,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:06:58,297 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:06:58,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:06:58,299 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:06:59,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:07:00,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:07:02,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:02,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:07:02,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:07:02,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:07:08,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:13,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:14,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:07:14,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:07:14,442 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:07:14,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:07:15,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:07:16,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:07:17,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:07:17,904 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:07:17,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:07:17,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:07:18,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:23,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:28,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:30,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:07:30,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:07:30,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:07:30,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:07:31,271 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:07:32,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:07:32,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:07:32,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:07:32,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:07:33,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:38,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:43,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:46,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:07:46,472 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:07:46,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:07:46,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:07:47,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:07:47,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:07:47,908 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:07:47,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:07:47,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:07:48,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:07:49,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:54,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:07:59,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:02,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:08:02,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:08:02,876 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:08:02,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:08:02,878 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:08:02,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:08:02,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:08:03,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:08:04,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:04,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:08:09,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:14,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:17,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:08:17,910 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:08:17,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:08:17,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:08:19,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:08:19,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:08:19,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:08:19,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:08:19,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:08:20,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:20,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:08:25,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:30,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:32,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:08:32,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:08:32,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:08:35,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:35,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:08:35,184 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:08:35,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:08:35,186 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:08:35,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:08:36,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:08:40,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:45,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:47,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:08:47,913 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:08:47,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:08:47,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:08:50,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:08:51,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:08:51,424 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:08:51,424 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:08:51,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:08:52,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:08:52,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:08:55,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:00,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:02,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:09:02,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:09:02,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:09:06,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:07,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:09:07,668 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:09:07,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:09:07,670 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:09:08,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:09:10,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:09:11,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:16,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:17,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:09:17,918 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:09:17,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:09:17,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:09:22,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:23,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:09:23,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:09:23,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:09:23,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:09:24,352 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:09:26,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:09:27,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:32,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:09:32,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:09:32,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:09:33,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:38,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:39,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:09:39,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:09:39,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:09:39,858 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:09:40,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:09:42,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:09:43,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:47,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:09:47,921 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:09:47,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:09:47,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:09:49,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:54,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:09:55,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:09:55,808 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:09:55,808 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:09:55,810 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:09:56,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:09:58,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:09:59,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:02,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:10:02,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:10:02,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:10:05,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:10,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:11,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:10:11,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:10:11,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:10:11,939 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:10:12,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:10:14,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:10:15,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:17,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:10:17,923 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:10:17,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:10:17,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:10:21,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:26,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:27,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:10:27,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:10:27,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:10:27,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:10:28,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:10:30,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:10:31,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:32,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:10:32,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:10:32,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:10:37,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:42,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:44,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:10:44,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:10:44,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:10:44,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:10:44,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:10:46,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:10:47,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:47,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:10:47,927 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:10:47,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:10:47,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:10:53,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:58,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:10:59,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:10:59,968 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:10:59,968 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:10:59,970 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:11:00,422 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:11:02,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:11:02,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:11:02,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:11:02,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:11:04,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:09,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:14,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:16,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:11:16,221 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:11:16,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:11:16,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:11:16,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:11:17,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:11:17,931 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:11:17,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:11:17,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:11:18,435 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:11:19,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:24,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:29,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:32,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:11:32,310 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:11:32,310 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:11:32,312 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:11:32,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:11:32,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:11:32,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:11:32,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:11:34,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:11:35,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:40,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:45,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:47,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:11:47,934 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:11:47,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:11:47,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:11:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:11:48,388 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:11:48,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:11:48,390 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:11:48,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:11:50,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:11:50,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:11:55,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:00,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:02,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:12:02,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:12:02,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:12:04,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:12:04,489 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:12:04,489 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:12:04,491 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:12:05,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:12:05,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:06,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:12:10,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:15,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:17,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:12:17,936 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:12:17,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:12:17,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:12:20,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:12:20,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:12:20,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:12:20,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:20,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:12:21,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:12:22,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:12:25,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:30,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:32,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:12:32,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:12:32,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:12:36,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:36,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:12:36,756 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:12:36,757 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:12:36,758 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:12:37,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:12:38,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:12:41,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:46,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:47,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:12:47,939 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:12:47,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:12:47,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:12:52,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:12:52,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:12:52,702 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:12:52,702 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:12:52,704 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:12:53,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:12:54,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:12:57,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:02,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:13:02,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:13:02,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:13:03,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:08,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:08,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:13:08,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:13:08,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:13:08,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:13:09,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:13:10,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:13:13,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:17,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:13:17,944 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:13:17,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:13:17,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:13:19,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:24,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:24,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:13:24,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:13:24,789 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:13:24,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:13:25,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:13:26,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:13:29,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:32,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:13:32,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:13:32,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:13:35,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:40,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:40,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:13:40,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:13:40,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:13:40,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:13:41,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:13:42,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:13:45,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:47,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:13:47,945 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:13:47,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:13:47,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:13:51,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:56,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:13:56,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:13:56,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:13:56,699 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:13:56,700 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:13:57,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:13:58,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:14:01,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:02,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:14:02,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:14:02,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:14:07,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:12,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:13,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:14:13,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:14:13,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:14:13,049 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:14:13,562 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:14:14,562 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:14:17,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:14:17,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:17,948 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:14:17,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:14:17,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:14:23,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:28,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:28,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:14:28,992 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:14:28,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:14:28,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:14:29,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:14:30,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:14:32,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:14:32,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:14:32,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:14:33,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:38,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:43,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:45,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:14:45,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:14:45,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:14:45,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:14:45,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:14:46,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:14:47,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:14:47,953 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:14:47,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:14:47,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:14:49,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:54,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:14:59,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:01,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:15:01,361 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:15:01,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:15:01,363 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:15:01,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:15:02,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:15:02,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:15:02,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:15:02,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:15:04,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:09,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:14,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:17,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:15:17,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:15:17,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:15:17,316 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:15:17,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:15:17,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:15:17,954 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:15:17,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:15:17,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:15:18,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:15:20,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:25,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:30,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:32,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:15:32,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:15:32,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:15:33,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:15:33,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:15:33,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:15:33,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:15:33,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:15:34,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:15:35,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:40,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:45,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:47,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:15:47,958 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:15:47,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:15:47,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:15:49,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:15:49,417 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:15:49,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:15:49,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:15:49,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:15:50,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:15:50,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:15:55,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:00,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:02,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:16:02,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:16:02,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:16:05,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:16:05,482 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:16:05,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:16:05,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:05,484 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:16:05,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:16:06,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:16:10,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:15,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:17,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:16:17,962 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:16:17,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:16:17,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:16:21,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:21,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:16:21,848 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:16:21,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:16:21,850 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:16:22,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:16:22,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:16:26,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:31,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:32,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:16:32,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:16:32,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:16:37,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:37,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:16:37,907 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:16:37,907 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:16:37,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:16:38,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:16:38,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:16:42,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:47,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:16:47,964 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:16:47,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:47,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:16:47,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:16:53,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:16:53,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:16:53,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:16:53,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:16:53,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:16:54,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:16:54,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:16:58,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:02,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:17:02,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:17:02,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:17:04,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:09,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:10,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:17:10,222 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:17:10,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:17:10,224 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:17:10,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:17:12,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:17:15,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:17,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:17:17,967 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:17:17,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:17:17,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:17:21,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:26,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:26,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:17:26,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:17:26,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:17:26,442 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:17:26,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:17:28,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:17:31,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:32,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:17:32,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:17:32,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:17:37,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:42,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:42,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:17:42,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:17:42,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:17:42,677 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:17:42,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:17:44,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:17:47,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:47,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:17:47,970 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:17:47,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:17:47,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:17:53,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:58,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:17:58,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:17:58,687 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:17:58,688 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:17:58,690 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:17:58,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:18:00,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:18:02,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:18:02,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:18:02,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:18:03,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:08,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:13,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:14,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:18:14,770 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:18:14,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:18:14,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:18:15,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:18:16,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:18:17,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:18:17,975 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:18:17,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:18:17,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:18:19,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:24,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:29,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:30,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:18:30,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:18:30,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:18:30,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:18:31,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:18:32,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:18:32,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:18:32,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:18:32,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:18:34,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:39,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:44,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:46,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:18:46,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:18:46,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:18:46,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:18:47,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:18:47,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:18:47,977 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:18:47,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:18:47,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:18:48,760 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:18:49,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:54,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:18:59,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:02,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:19:02,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:19:02,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:19:03,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:19:03,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:19:03,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:19:03,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:19:03,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:19:04,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:04,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:09,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:14,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:17,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:19:17,980 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:19:17,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:19:17,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:19:19,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:19:19,264 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:19:19,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:19:19,266 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:19:19,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:19:20,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:20,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:25,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:26,788 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:28,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:30,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:30,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:32,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:19:33,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:19:33,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:19:36,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:36,795 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:38,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:41,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:19:42,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:42,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:19:47,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:47,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:19:47,982 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:19:48,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:19:48,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:19:52,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:57,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:19:57,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:19:57,297 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:19:57,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:19:57,299 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:19:57,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:19:58,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:20:02,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:02,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:20:03,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:20:03,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:20:08,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:13,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:13,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:20:13,403 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:20:13,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:20:13,405 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:20:13,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:20:14,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:20:17,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:20:17,989 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:20:18,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:20:18,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:20:18,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:23,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:28,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:29,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:20:29,703 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:20:29,704 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:20:29,705 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:20:29,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:20:30,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:20:32,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:20:33,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:20:33,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:20:33,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:38,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:43,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:45,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:20:45,973 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:20:45,973 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:20:45,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:20:46,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:20:46,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:20:47,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:20:47,989 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:20:48,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:20:48,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:20:49,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:54,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:20:59,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:02,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:21:02,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:21:02,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:21:02,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:21:02,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:21:02,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:21:02,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:21:03,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:21:03,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:21:04,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:09,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:14,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:17,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:21:17,992 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:21:18,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:21:18,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:21:18,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:21:18,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:21:18,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:21:18,248 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:21:18,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:21:18,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:21:19,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:24,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:29,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:32,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:21:33,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:21:33,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:21:34,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:21:34,271 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:21:34,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:21:34,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:21:34,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:21:34,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:21:35,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:40,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:45,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:47,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:21:47,997 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:21:48,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:21:48,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:21:50,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:21:50,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:21:50,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:21:50,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:21:50,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:21:50,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:21:51,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:21:56,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:01,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:02,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:22:03,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:22:03,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:22:06,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:22:06,269 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:22:06,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:22:06,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:06,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:22:06,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:22:06,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:22:11,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:16,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:17,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:22:17,998 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:22:18,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:22:18,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:22:22,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:22:22,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:22:22,132 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:22:22,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:22,134 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:22:22,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:22:22,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:22:27,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:32,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:32,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:22:33,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:22:33,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:22:37,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:38,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:22:38,186 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:22:38,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:22:38,187 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:22:38,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:22:38,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:22:43,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:47,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:22:48,002 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:22:48,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:22:48,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:22:48,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:53,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:22:54,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:22:54,154 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:22:54,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:22:54,156 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:22:54,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:22:54,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:22:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:02,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:23:03,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:23:03,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:23:04,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:09,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:10,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:23:10,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:23:10,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:23:10,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:23:10,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:23:10,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:23:15,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:17,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:23:18,006 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:23:18,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:23:18,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:23:20,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:25,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:26,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:23:26,128 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:23:26,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:23:26,130 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:23:26,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:23:26,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:23:31,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:32,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:23:33,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:23:33,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:23:36,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:41,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:42,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:23:42,293 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:23:42,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:23:42,296 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:23:42,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:23:42,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:23:46,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:47,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:23:48,008 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:23:48,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:23:48,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:23:52,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:57,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:23:58,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:23:58,149 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:23:58,150 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:23:58,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:23:58,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:23:58,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:24:02,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:02,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:24:03,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:24:03,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:24:08,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:13,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:14,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:24:14,210 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:24:14,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:24:14,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:24:15,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:24:15,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:24:17,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:24:18,011 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:24:18,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:24:18,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:24:18,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:23,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:28,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:30,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:24:30,405 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:24:30,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:24:30,407 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:24:31,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:24:31,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:24:32,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:24:33,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:24:33,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:24:34,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:39,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:44,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:46,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:24:46,449 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:24:46,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:24:46,451 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:24:47,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:24:47,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:24:47,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:24:48,014 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:24:48,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:24:48,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:24:49,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:54,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:24:59,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:02,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:25:02,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:25:02,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:25:02,412 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:25:02,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:25:03,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:25:03,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:25:03,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:25:03,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:25:05,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:10,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:15,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:17,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:25:18,016 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:25:18,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:25:18,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:25:18,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:25:18,367 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:25:18,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:25:18,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:25:19,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:25:19,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:25:20,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:25,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:30,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:32,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:25:33,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:25:33,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:25:34,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:25:34,547 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:25:34,547 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:25:34,549 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:25:35,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:25:35,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:25:35,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:40,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:45,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:47,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:25:48,019 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:25:48,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:25:48,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:25:50,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:25:50,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:25:50,699 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:25:50,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:25:50,701 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:25:51,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:25:53,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:25:55,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:00,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:02,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:26:03,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:26:03,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:26:06,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:06,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:26:06,981 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:26:06,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:26:06,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:26:07,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:26:09,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:26:11,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:16,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:17,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:26:18,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:26:18,023 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:26:18,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:26:22,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:22,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:26:22,843 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:26:22,843 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:26:22,845 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:26:23,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:26:25,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:26:27,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:32,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:26:33,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:26:33,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:26:33,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:38,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:38,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:26:38,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:26:38,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:26:38,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:26:39,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:26:41,111 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:26:43,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:47,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:26:48,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:26:48,028 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:26:48,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:26:49,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:54,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:26:55,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:26:55,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:26:55,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:26:55,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:26:56,122 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:26:57,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:26:59,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:02,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:27:03,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:27:03,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:27:05,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:10,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:11,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:27:11,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:27:11,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:27:11,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:27:12,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:27:13,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:27:15,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:17,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:27:18,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:27:18,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:27:18,239 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:27:21,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:26,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:27,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:27:27,561 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:27:27,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:27:27,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:27:28,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:27:29,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:27:31,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:32,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:27:33,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:27:33,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:27:37,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:42,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:43,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:27:43,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:27:43,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:27:43,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:27:44,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:27:45,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:27:47,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:47,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:27:48,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:27:48,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:27:48,206 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:27:53,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:58,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:27:59,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:27:59,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:27:59,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:27:59,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:28:00,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:28:01,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:28:02,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:28:03,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:28:03,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:28:03,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:08,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:13,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:15,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:28:15,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:28:15,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:28:15,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:28:16,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:28:17,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:28:17,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:28:18,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:28:18,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:28:18,234 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:28:19,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:24,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:29,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:31,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:28:31,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:28:31,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:28:31,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:28:32,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:28:32,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:28:33,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:28:33,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:28:33,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:28:34,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:39,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:44,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:47,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:28:48,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:28:48,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:28:48,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:28:48,243 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:28:48,243 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:28:48,244 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:28:48,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:28:49,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:28:49,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:28:50,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:28:55,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:00,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:02,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:29:03,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:29:03,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:29:04,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:29:04,294 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:29:04,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:29:04,297 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:29:05,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:29:05,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:29:05,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:10,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:15,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:17,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:29:18,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:29:18,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:29:18,235 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:29:20,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:29:20,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:29:20,558 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:29:20,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:20,559 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:29:21,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:29:21,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:29:25,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:30,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:32,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:29:33,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:29:33,023 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:29:36,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:36,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:29:36,514 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:29:36,514 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:29:36,517 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:29:37,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:29:37,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:29:41,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:46,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:47,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:29:48,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:29:48,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:29:48,296 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:29:52,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:29:52,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:29:52,635 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:29:52,636 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:29:52,637 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:29:53,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:29:53,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:29:57,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:02,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:30:03,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:30:03,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:30:03,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:08,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:08,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:30:08,690 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:30:08,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:30:08,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:30:09,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:30:09,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:30:13,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:17,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:30:18,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:30:18,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:30:18,254 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:30:19,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:24,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:24,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:30:24,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:30:24,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:30:24,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:30:25,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:30:25,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:30:29,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:32,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:30:33,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:30:33,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:30:35,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:40,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:40,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:30:40,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:30:40,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:30:40,684 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:30:41,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:30:41,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:30:45,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:47,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:30:48,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:30:48,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:30:48,251 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:30:51,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:56,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:30:56,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:30:56,806 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:30:56,807 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:30:56,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:30:57,302 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:30:57,302 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:31:01,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:02,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:31:03,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:31:03,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:31:07,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:12,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:12,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:31:12,859 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:31:12,859 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:31:12,862 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:31:13,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:31:13,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:31:17,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:17,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:31:18,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:31:18,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:31:18,192 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:31:23,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:28,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:28,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:31:28,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:31:28,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:31:28,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:31:29,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:31:29,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:31:32,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:31:33,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:31:33,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:31:33,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:38,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:43,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:45,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:31:45,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:31:45,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:31:45,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:31:45,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:31:45,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:31:47,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:31:48,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:31:48,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:31:48,194 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:31:49,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:54,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:31:59,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:01,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:32:01,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:32:01,090 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:32:01,092 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:32:01,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:32:02,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:32:03,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:32:03,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:32:03,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:32:04,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:09,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:14,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:17,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:32:17,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:32:17,113 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:32:17,116 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:32:17,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:32:17,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:32:18,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:32:18,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:32:18,300 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:32:19,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:19,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:32:24,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:29,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:32:32,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:32:32,965 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:32:32,966 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:32:32,967 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:32:33,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:32:33,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:32:33,370 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:32:33,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:32:35,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:40,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:45,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:47,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:32:48,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:32:48,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:32:48,282 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:32:49,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:32:49,119 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:32:49,120 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:32:49,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:32:49,382 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:32:51,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:32:51,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:32:56,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:01,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:33:03,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:33:03,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:33:05,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:33:05,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:33:05,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:33:05,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:33:05,395 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:33:06,396 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:33:07,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:12,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:17,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:17,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:33:18,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:33:18,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:33:18,222 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:33:21,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:33:21,043 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:33:21,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:33:21,046 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:33:21,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:33:22,408 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:33:23,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:28,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:33:33,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:33:33,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:33:33,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:37,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:33:37,014 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:33:37,015 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:33:37,016 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:33:37,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:33:38,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:33:39,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:44,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:47,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:33:48,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:33:48,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:33:48,280 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:33:49,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:53,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:33:53,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:33:53,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:33:53,273 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:33:53,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:33:55,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:33:56,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:34:00,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:34:03,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:34:03,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:34:06,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:09,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:34:09,139 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:34:09,139 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:34:09,141 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:34:09,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:34:10,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:34:12,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:17,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:34:18,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:34:18,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:34:18,294 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:34:22,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:25,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:34:25,195 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:34:25,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:34:25,198 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:34:25,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:34:26,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:34:28,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:32,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:34:33,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:34:33,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:34:33,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:38,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:41,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:34:41,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:34:41,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:34:41,473 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:34:42,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:34:43,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:44,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:34:47,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:34:48,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:34:48,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:34:48,223 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:34:49,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:54,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:34:57,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:34:57,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:34:57,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:34:57,500 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:34:58,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:34:59,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:00,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:35:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:35:03,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:35:03,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:35:05,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:10,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:13,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:35:13,351 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:35:13,352 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:35:13,354 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:35:13,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:35:15,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:16,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:35:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:35:18,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:35:18,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:35:18,221 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:35:21,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:26,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:29,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:35:29,524 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:35:29,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:35:29,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:35:30,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:35:31,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:32,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:35:32,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:35:33,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:35:33,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:35:37,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:42,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:45,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:35:45,593 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:35:45,593 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:35:45,595 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:35:46,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:35:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:35:48,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:35:48,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:35:48,297 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:35:48,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:35:53,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:35:58,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:01,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:36:01,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:36:01,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:36:01,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:36:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:36:02,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:36:03,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:36:03,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:36:04,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:04,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:36:09,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:14,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:36:18,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:36:18,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:36:18,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:36:18,237 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:36:18,238 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:36:18,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:36:18,239 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:36:18,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:36:19,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:20,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:36:24,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:29,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:36:33,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:36:33,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:36:34,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:36:34,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:36:34,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:36:34,055 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:36:34,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:36:35,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:36,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:36:40,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:45,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:47,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:36:48,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:36:48,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:36:48,288 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:36:50,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:36:50,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:36:50,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:36:50,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:36:50,244 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:36:50,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:36:52,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:36:55,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:00,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:02,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:37:03,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:37:03,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:37:05,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:06,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:37:06,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:37:06,580 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:37:06,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:37:07,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:37:08,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:37:10,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:15,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:17,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:37:18,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:37:18,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:37:18,280 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:37:21,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:22,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:37:22,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:37:22,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:37:22,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:37:23,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:37:24,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:37:26,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:31,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:37:33,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:37:33,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:37:37,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:38,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:37:38,908 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:37:38,908 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:37:38,910 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:37:39,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:37:40,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:37:42,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:47,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:37:48,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:37:48,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:37:48,246 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:37:48,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:53,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:37:55,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:37:55,151 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:37:55,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:37:55,153 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:37:55,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:37:56,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:37:59,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:38:03,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:38:03,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:38:04,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:09,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:11,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:38:11,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:38:11,540 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:38:11,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:38:11,618 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:38:14,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:14,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:38:17,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:38:18,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:38:18,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:38:18,284 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:38:20,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:25,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:27,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:38:27,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:38:27,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:38:27,673 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:38:28,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:38:30,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:38:30,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:38:33,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:38:33,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:38:36,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:41,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:43,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:38:43,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:38:43,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:38:43,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:38:44,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:38:46,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:38:46,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:47,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:38:48,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:38:48,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:38:48,264 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:38:52,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:38:57,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:00,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:39:00,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:39:00,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:39:00,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:39:00,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:39:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:02,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:39:02,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:39:03,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:39:03,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:39:08,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:13,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:16,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:39:16,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:39:16,145 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:39:16,148 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:39:16,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:39:17,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:39:18,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:39:18,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:39:18,210 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:39:18,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:39:19,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:24,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:29,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:32,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:39:32,368 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:39:32,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:39:32,370 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:39:32,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:39:32,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:39:33,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:39:33,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:39:34,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:34,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:39:39,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:44,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:39:48,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:39:48,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:39:48,252 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:39:48,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:39:48,328 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:39:48,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:39:48,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:39:48,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:39:49,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:50,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:39:54,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:39:59,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:02,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:40:03,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:40:03,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:40:04,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:40:04,480 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:40:04,481 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:40:04,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:04,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:40:04,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:40:06,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:40:09,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:14,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:17,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:40:18,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:40:18,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:40:18,203 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:40:20,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:20,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:40:20,419 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:40:20,419 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:40:20,421 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:40:20,713 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:40:22,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:40:25,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:30,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:32,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:40:33,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:40:33,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:40:36,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:36,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:40:36,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:40:36,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:40:36,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:40:36,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:40:38,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:40:41,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:46,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:40:48,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:40:48,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:40:48,243 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:40:52,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:40:52,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:40:52,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:40:52,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:40:52,500 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:40:52,739 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:40:54,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:40:57,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:02,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:41:03,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:41:03,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:41:03,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:08,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:08,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:41:08,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:41:08,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:41:08,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:41:09,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:41:10,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:41:13,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:17,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:41:18,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:41:18,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:41:18,215 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:41:19,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:24,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:24,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:41:24,839 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:41:24,840 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:41:24,842 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:41:25,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:41:26,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:41:29,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:32,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:41:33,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:41:33,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:41:35,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:40,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:40,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:41:40,943 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:41:40,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:41:40,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:41:41,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:41:42,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:41:45,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:47,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:41:48,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:41:48,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:41:48,245 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:41:51,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:56,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:41:56,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:41:56,994 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:41:56,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:41:56,997 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:41:57,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:41:58,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:42:01,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:02,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:42:03,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:42:03,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:42:07,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:12,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:12,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:42:12,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:42:12,949 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:42:12,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:42:13,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:42:14,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:42:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:42:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:18,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:42:18,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:42:18,207 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:42:23,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:28,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:29,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:42:29,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:42:29,082 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:42:29,085 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:42:29,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:42:30,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:42:32,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:42:33,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:42:33,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:42:33,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:38,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:43,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:45,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:42:45,406 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:42:45,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:42:45,408 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:42:45,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:42:46,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:42:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:42:48,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:42:48,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:42:48,210 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:42:49,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:54,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:42:59,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:01,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:43:01,464 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:43:01,465 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:43:01,467 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:43:01,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:43:02,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:43:02,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:43:03,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:43:03,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:43:06,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:11,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:16,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:17,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:43:17,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:43:17,506 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:43:17,507 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:43:17,508 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:43:17,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:43:18,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:43:18,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:43:18,211 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:43:18,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:43:21,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:26,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:31,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:32,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:43:33,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:43:33,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:43:33,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:43:33,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:43:33,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:43:33,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:43:33,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:43:34,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:43:36,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:41,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:46,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:43:48,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:43:48,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:43:48,270 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:43:49,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:43:49,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:43:49,327 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:43:49,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:43:49,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:43:50,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:43:52,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:43:57,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:02,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:02,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:44:03,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:44:03,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:44:05,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:44:05,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:44:05,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:44:05,606 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:44:05,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:44:06,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:44:07,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:12,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:17,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:44:18,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:44:18,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:44:18,254 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:44:18,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:21,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:44:21,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:44:21,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:44:21,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:44:21,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:44:22,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:44:23,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:28,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:44:33,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:44:33,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:44:34,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:37,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:44:37,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:44:37,702 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:44:37,704 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:44:37,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:44:38,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:44:39,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:44,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:44:48,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:44:48,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:44:48,326 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:44:50,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:44:53,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:44:53,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:44:53,768 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:44:53,769 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:44:53,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:44:54,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:44:55,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:00,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:45:03,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:45:03,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:45:06,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:10,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:45:10,099 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:45:10,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:45:10,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:45:10,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:45:12,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:12,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:45:17,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:17,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:45:18,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:45:18,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:45:18,250 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:45:22,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:25,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:45:25,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:45:25,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:45:25,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:45:25,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:45:27,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:28,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:45:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:45:33,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:45:33,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:45:33,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:38,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:42,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:45:42,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:45:42,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:45:42,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:45:42,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:45:44,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:44,998 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:45:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:45:48,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:45:48,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:45:48,250 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:45:49,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:54,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:45:58,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:45:58,227 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:45:58,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:45:58,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:45:59,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:46:00,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:01,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:46:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:46:03,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:46:03,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:46:05,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:10,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:14,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:46:14,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:46:14,566 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:46:14,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:46:15,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:46:15,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:17,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:46:17,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:46:18,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:46:18,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:46:18,273 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:46:21,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:26,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:30,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:46:30,769 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:46:30,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:46:30,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:46:31,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:46:31,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:32,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:46:33,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:46:33,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:46:33,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:46:37,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:42,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:46,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:46:46,845 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:46:46,845 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:46:46,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:46:47,051 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:46:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:47,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:46:48,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:46:48,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:46:48,259 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:46:49,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:46:53,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:46:58,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:02,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:47:02,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:47:03,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:47:03,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:47:03,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:47:03,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:47:03,042 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:47:03,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:47:03,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:05,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:47:08,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:13,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:17,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:47:18,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:47:18,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:47:18,295 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:47:18,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:19,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:47:19,331 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:47:19,332 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:47:19,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:47:20,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:47:21,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:47:23,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:28,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:32,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:47:33,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:47:33,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:47:34,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:35,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:47:35,286 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:47:35,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:47:35,288 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:47:36,093 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:47:37,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:47:39,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:44,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:47,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:47:48,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:47:48,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:47:48,307 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:47:49,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:51,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:47:51,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:47:51,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:47:51,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:47:52,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:47:53,107 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:47:54,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:47:59,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:02,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:48:03,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:48:03,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:48:05,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:07,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:48:07,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:48:07,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:48:07,692 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:48:08,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:48:09,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:48:10,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:15,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:17,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:48:18,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:48:18,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:48:18,271 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:48:21,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:23,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:48:23,953 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:48:23,954 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:48:23,956 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:48:24,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:48:25,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:48:26,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:31,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:32,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:48:33,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:48:33,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:48:37,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:40,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:48:40,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:48:40,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:48:40,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:48:40,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:48:41,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:48:43,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:47,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:48:48,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:48:48,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:48:48,212 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:48:48,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:53,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:48:56,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:48:56,095 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:48:56,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:48:56,098 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:48:56,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:48:57,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:48:59,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:02,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:49:03,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:49:03,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:49:04,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:09,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:12,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:49:12,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:49:12,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:49:12,427 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:49:13,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:49:14,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:15,176 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:49:17,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:49:18,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:49:18,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:49:18,281 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:49:20,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:25,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:28,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:49:28,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:49:28,549 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:49:28,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:49:29,188 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:49:30,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:31,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:49:32,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:49:33,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:49:33,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:49:36,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:41,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:44,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:49:44,814 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:49:44,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:49:44,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:49:45,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:49:46,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:47,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:49:47,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:49:48,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:49:48,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:49:48,254 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:49:52,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:49:57,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:00,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:50:00,782 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:50:00,782 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:50:00,784 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:50:01,216 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:50:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:50:03,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:50:03,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:50:03,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:50:08,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:13,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:16,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:50:16,913 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:50:16,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:50:16,915 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:50:17,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:50:17,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:50:18,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:50:18,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:50:18,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:50:18,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:19,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:50:23,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:28,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:32,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:50:32,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:50:32,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:50:32,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:50:32,863 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:50:33,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:50:33,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:50:33,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:50:34,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:35,245 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:50:39,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:44,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:47,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:50:48,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:50:48,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:50:48,296 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:50:49,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:50:49,024 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:50:49,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:50:49,026 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:50:49,257 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:50:50,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:50:51,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:50:55,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:00,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:02,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:51:03,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:51:03,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:51:05,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:51:05,115 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:51:05,116 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:51:05,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:05,119 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:51:05,271 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:51:07,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:51:10,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:15,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:51:18,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:51:18,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:51:18,294 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:51:20,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:21,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:51:21,245 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:51:21,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:51:21,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:51:21,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:51:23,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:51:26,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:31,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:51:33,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:51:33,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:51:36,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:37,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:51:37,308 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:51:37,309 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:51:37,311 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:51:38,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:51:39,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:51:41,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:46,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:47,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:51:48,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:51:48,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:51:48,237 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:51:52,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:51:53,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:51:53,488 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:51:53,488 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:51:53,490 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:51:54,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:51:55,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:51:57,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:02,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:52:03,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:52:03,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:52:03,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:08,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:09,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:52:09,515 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:52:09,515 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:52:09,518 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:52:10,326 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:52:11,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:52:13,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:52:18,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:52:18,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:52:18,288 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:52:19,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:24,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:25,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:52:25,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:52:25,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:52:25,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:52:26,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:52:27,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:52:29,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:52:33,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:52:33,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:52:35,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:40,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:41,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:52:41,753 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:52:41,754 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:52:41,756 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:52:42,357 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:52:43,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:52:45,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:47,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:52:48,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:52:48,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:52:48,270 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:52:51,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:56,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:52:57,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:52:57,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:52:57,600 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:52:57,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:52:58,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:52:59,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:53:01,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:02,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:53:03,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:53:03,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:53:07,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:12,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:13,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:53:13,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:53:13,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:53:13,754 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:53:14,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:53:15,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:53:17,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:53:18,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:53:18,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:53:18,266 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:53:23,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:28,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:29,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:53:29,708 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:53:29,708 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:53:29,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:53:30,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:53:31,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:53:32,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:53:33,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:53:33,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:53:33,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:38,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:43,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:45,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:53:45,764 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:53:45,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:53:45,766 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:53:46,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:53:47,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:53:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:53:48,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:53:48,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:53:48,286 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:53:48,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:53,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:53:58,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:01,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:54:01,740 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:54:01,740 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:54:01,742 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:54:02,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:54:02,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:54:03,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:54:03,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:54:03,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:54:04,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:09,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:14,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:54:17,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:54:17,899 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:54:17,900 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:54:17,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:54:18,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:54:18,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:54:18,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:54:18,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:54:19,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:19,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:54:24,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:29,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:32,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:54:33,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:54:33,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:54:33,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:54:33,858 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:54:33,858 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:54:33,860 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:54:34,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:54:34,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:35,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:54:39,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:44,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:54:48,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:54:48,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:54:48,245 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:54:50,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:54:50,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:54:50,013 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:54:50,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:54:50,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:54:50,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:54:51,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:54:55,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:00,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:02,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:55:03,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:55:03,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:55:05,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:06,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:55:06,277 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:55:06,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:55:06,279 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:55:06,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:55:07,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:55:10,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:15,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:17,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:55:18,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:55:18,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:55:18,257 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:55:21,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:22,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:55:22,334 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:55:22,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:55:22,336 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:55:22,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:55:23,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:55:26,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:31,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:32,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:55:33,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:55:33,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:55:37,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:38,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:55:38,286 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:55:38,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:55:38,288 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:55:38,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:55:39,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:55:42,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:47,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:55:48,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:55:48,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:55:48,268 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:55:53,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:55:54,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:55:54,241 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:55:54,241 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:55:54,242 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:55:54,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:55:55,521 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:55:59,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:56:03,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:56:03,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:56:04,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:09,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:10,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:56:10,390 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:56:10,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:56:10,392 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:56:10,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:56:11,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:56:14,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:17,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:56:18,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:56:18,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:56:18,251 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:56:20,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:25,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:26,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:56:26,648 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:56:26,649 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:56:26,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:56:27,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:56:27,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:56:30,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:56:33,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:56:33,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:56:36,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:41,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:42,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:56:42,713 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:56:42,713 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:56:42,715 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:56:43,562 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:56:45,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:56:46,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:47,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:56:48,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:56:48,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:56:48,259 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:56:52,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:57,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:56:58,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:56:58,777 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:56:58,778 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:56:58,780 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:56:59,576 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:57:01,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:57:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:57:03,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:57:03,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:57:08,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:13,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:14,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:57:14,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:57:14,820 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:57:14,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:57:15,589 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:57:17,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:57:17,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:57:18,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:57:18,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:57:18,268 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:57:18,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:23,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:28,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:31,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:57:31,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:57:31,050 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:57:31,052 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:57:31,602 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:57:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:57:33,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:57:33,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:57:33,603 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:57:34,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:39,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:44,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:47,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:57:47,211 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:57:47,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:57:47,213 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:57:47,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:57:47,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:57:48,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:57:48,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:57:48,257 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:57:49,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:49,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:57:54,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:57:59,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:58:03,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:58:03,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:58:03,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:58:03,337 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:58:03,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:58:03,340 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:58:03,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:58:04,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:05,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:58:09,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:14,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:58:18,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:58:18,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:58:18,260 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:58:19,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:58:19,474 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:58:19,474 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:58:19,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:19,476 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:58:19,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:58:21,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:58:24,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:29,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:58:33,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:58:33,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:58:35,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:35,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:58:35,742 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:58:35,743 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:58:35,745 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:58:36,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:58:37,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:58:40,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:45,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:58:48,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:58:48,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:58:48,285 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:58:51,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:58:51,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:58:51,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:58:51,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:58:51,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:58:52,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:58:53,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:58:56,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:01,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:02,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:59:03,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:59:03,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:59:07,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:07,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:59:07,910 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:59:07,910 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:59:07,912 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:59:08,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:59:09,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:59:12,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:59:18,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:59:18,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:59:18,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:18,268 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:59:23,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:24,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:59:24,240 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:59:24,240 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:59:24,241 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:59:24,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:59:25,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:59:29,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:32,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:59:33,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:59:33,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:59:35,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:40,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:40,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:59:40,391 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:59:40,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:59:40,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:59:40,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:59:41,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-28 23:59:45,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-28 23:59:48,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-28 23:59:48,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-28 23:59:48,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-28 23:59:51,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:56,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-28 23:59:56,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-28 23:59:56,562 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-28 23:59:56,562 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-28 23:59:56,564 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-28 23:59:56,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-28 23:59:57,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:00:01,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:02,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:00:03,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:00:03,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:00:07,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:12,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:12,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:00:12,640 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:00:12,641 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:00:12,642 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:00:12,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:00:13,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:00:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:00:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:18,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:00:18,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:00:18,274 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:00:23,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:28,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:28,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:00:28,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:00:28,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:00:28,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:00:29,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:00:29,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:00:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:00:33,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:00:33,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:00:34,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:39,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:44,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:44,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:00:44,812 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:00:44,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:00:44,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:00:45,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:00:45,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:00:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:00:48,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:00:48,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:00:48,279 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:00:49,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:54,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:00:59,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:01,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:01:01,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:01:01,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:01:01,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:01:01,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:01:02,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:01:03,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:01:03,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:01:03,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:01:05,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:10,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:15,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:17,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:01:17,080 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:01:17,080 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:01:17,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:01:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:01:17,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:01:18,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:01:18,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:01:18,280 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:01:19,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:01:21,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:26,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:31,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:01:33,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:01:33,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:01:33,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:01:33,259 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:01:33,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:01:33,261 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:01:33,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:01:35,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:01:37,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:42,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:47,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:47,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:01:48,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:01:48,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:01:48,311 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:01:49,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:01:49,290 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:01:49,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:01:49,292 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:01:49,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:01:51,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:01:52,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:01:57,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:02,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:02,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:02:03,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:02:03,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:02:05,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:02:05,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:02:05,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:02:05,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:02:05,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:02:07,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:07,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:02:12,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:02:18,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:02:18,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:02:18,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:18,314 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:02:21,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:02:21,531 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:02:21,531 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:02:21,532 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:02:21,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:02:23,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:23,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:02:28,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:32,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:02:33,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:02:33,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:02:34,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:37,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:02:37,379 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:02:37,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:02:37,381 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:02:37,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:02:39,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:39,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:02:44,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:47,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:02:48,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:02:48,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:02:48,319 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:02:50,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:53,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:02:53,499 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:02:53,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:02:53,501 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:02:53,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:02:55,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:02:55,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:03:00,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:02,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:03:03,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:03:03,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:03:06,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:09,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:03:09,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:03:09,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:03:09,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:03:09,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:03:11,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:11,888 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:03:16,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:17,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:03:18,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:03:18,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:03:18,320 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:03:22,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:25,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:03:25,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:03:25,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:03:25,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:03:25,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:03:27,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:27,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:03:32,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:03:33,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:03:33,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:03:33,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:38,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:41,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:03:41,572 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:03:41,572 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:03:41,574 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:03:41,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:03:43,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:43,914 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:03:47,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:03:48,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:03:48,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:03:48,323 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:03:49,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:54,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:57,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:03:57,761 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:03:57,762 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:03:57,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:03:57,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:03:59,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:03:59,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:04:02,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:04:03,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:04:03,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:04:05,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:10,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:13,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:04:13,618 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:04:13,619 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:04:13,620 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:04:13,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:04:15,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:15,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:04:17,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:04:18,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:04:18,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:04:18,327 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:04:21,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:26,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:29,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:04:29,865 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:04:29,865 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:04:29,867 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:04:29,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:04:31,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:31,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:04:32,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:04:33,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:04:33,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:04:37,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:42,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:45,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:04:45,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:04:45,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:04:45,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:04:45,970 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:04:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:04:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:47,971 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:04:48,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:04:48,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:04:48,329 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:04:53,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:04:58,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:02,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:05:02,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:05:02,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:05:02,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:05:02,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:05:02,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:05:03,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:05:03,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:05:03,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:03,984 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:05:08,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:13,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:05:18,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:05:18,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:05:18,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:05:18,342 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:05:18,342 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:05:18,344 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:05:18,345 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:05:18,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:05:19,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:19,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:05:24,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:29,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:32,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:05:33,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:05:33,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:05:34,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:05:34,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:05:34,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:05:34,287 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:05:35,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:05:35,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:36,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:05:40,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:45,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:05:48,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:05:48,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:05:48,336 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:05:50,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:05:50,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:05:50,236 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:05:50,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:05:51,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:05:51,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:05:52,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:05:56,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:01,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:02,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:06:03,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:06:03,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:06:06,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:06:06,205 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:06:06,205 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:06:06,207 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:06:07,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:06:07,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:08,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:06:12,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:17,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:06:18,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:06:18,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:06:18,339 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:06:22,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:22,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:06:22,369 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:06:22,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:06:22,372 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:06:23,051 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:06:24,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:06:27,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:32,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:32,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:06:33,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:06:33,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:06:38,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:38,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:06:38,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:06:38,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:06:38,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:06:39,064 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:06:40,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:06:43,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:47,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:06:48,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:06:48,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:06:48,342 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:06:49,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:54,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:06:54,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:06:54,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:06:54,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:06:54,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:06:55,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:06:56,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:06:59,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:02,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:07:03,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:07:03,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:07:05,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:10,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:10,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:07:10,597 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:07:10,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:07:10,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:07:11,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:07:12,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:07:15,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:17,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:07:18,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:07:18,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:07:18,345 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:07:21,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:26,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:26,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:07:26,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:07:26,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:07:26,829 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:07:27,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:07:28,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:07:31,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:32,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:07:33,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:07:33,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:07:37,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:42,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:42,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:07:42,992 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:07:42,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:07:42,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:07:43,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:07:44,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:07:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:07:48,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:07:48,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:07:48,349 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:07:53,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:58,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:07:59,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:07:59,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:07:59,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:07:59,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:07:59,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:08:00,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:08:02,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:08:03,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:08:03,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:08:04,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:09,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:14,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:15,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:08:15,119 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:08:15,120 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:08:15,121 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:08:15,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:08:16,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:08:17,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:08:18,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:08:18,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:08:18,351 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:08:19,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:24,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:29,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:31,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:08:31,150 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:08:31,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:08:31,152 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:08:31,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:08:32,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:08:32,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:08:33,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:08:33,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:08:35,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:40,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:45,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:47,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:08:47,310 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:08:47,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:08:47,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:08:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:08:48,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:08:48,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:08:48,191 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:08:48,191 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:08:48,354 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:08:50,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:08:55,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:00,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:02,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:09:03,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:09:03,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:09:03,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:09:03,455 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:09:03,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:09:03,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:09:04,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:09:04,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:09:05,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:10,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:15,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:17,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:09:18,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:09:18,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:09:18,357 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:09:19,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:09:19,377 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:09:19,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:09:19,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:09:20,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:09:20,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:09:21,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:26,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:31,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:32,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:09:33,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:09:33,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:09:35,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:09:35,912 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:09:35,913 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:09:35,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:09:36,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:09:36,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:38,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:09:41,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:46,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:47,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:09:48,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:09:48,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:09:48,360 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:09:51,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:09:51,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:09:51,969 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:09:51,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:09:51,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:09:52,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:09:54,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:09:56,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:01,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:02,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:10:03,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:10:03,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:10:07,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:08,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:10:08,223 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:10:08,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:10:08,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:10:08,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:10:10,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:10:13,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:10:18,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:10:18,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:10:18,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:18,363 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:10:23,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:24,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:10:24,189 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:10:24,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:10:24,191 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:10:24,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:10:26,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:10:29,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:10:33,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:10:33,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:10:34,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:39,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:40,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:10:40,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:10:40,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:10:40,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:10:41,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:10:42,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:10:44,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:10:48,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:10:48,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:10:48,365 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:10:50,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:55,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:10:56,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:10:56,429 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:10:56,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:10:56,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:10:57,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:10:58,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:11:00,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:02,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:11:03,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:11:03,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:11:06,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:11,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:12,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:11:12,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:11:12,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:11:12,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:11:13,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:11:14,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:11:16,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:11:18,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:11:18,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:11:18,369 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:11:22,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:27,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:28,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:11:28,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:11:28,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:11:28,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:11:29,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:11:30,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:11:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:11:33,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:11:33,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:11:38,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:43,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:44,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:11:44,942 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:11:44,942 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:11:44,944 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:11:45,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:11:46,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:11:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:11:48,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:11:48,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:11:48,371 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:11:48,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:53,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:11:58,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:01,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:12:01,109 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:12:01,110 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:12:01,112 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:12:01,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:12:02,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:12:02,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:12:03,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:12:03,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:12:04,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:09,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:14,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:17,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:12:17,343 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:12:17,343 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:12:17,345 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:12:17,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:12:17,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:12:18,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:12:18,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:12:18,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:12:18,374 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:12:19,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:24,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:29,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:32,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:12:33,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:12:33,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:12:33,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:12:33,356 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:12:33,356 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:12:33,358 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:12:33,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:12:34,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:12:35,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:40,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:45,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:47,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:12:48,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:12:48,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:12:48,378 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:12:49,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:12:49,205 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:12:49,205 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:12:49,207 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:12:49,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:12:50,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:12:51,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:12:56,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:01,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:02,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:13:03,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:13:03,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:13:05,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:13:05,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:13:05,353 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:13:05,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:13:05,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:13:06,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:06,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:11,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:16,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:17,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:13:18,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:13:18,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:13:18,380 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:13:21,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:13:21,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:13:21,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:13:21,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:13:21,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:13:22,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:22,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:26,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:27,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:30,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:32,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:32,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:13:33,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:13:33,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:13:33,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:36,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:38,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:38,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:40,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:13:42,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:13:43,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:47,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:13:48,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:13:48,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:13:48,384 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:13:49,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:54,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:13:57,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:13:57,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:13:57,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:13:57,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:13:57,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:13:58,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:14:00,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:02,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:14:03,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:14:03,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:14:05,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:10,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:13,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:14:13,529 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:14:13,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:14:13,531 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:14:14,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:14:14,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:14:15,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:17,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:14:18,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:14:18,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:14:18,387 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:14:21,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:26,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:29,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:14:29,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:14:29,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:14:29,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:14:30,475 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:14:30,475 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:14:31,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:32,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:14:33,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:14:33,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:14:37,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:42,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:45,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:14:45,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:14:45,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:14:45,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:14:46,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:14:46,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:14:47,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:14:47,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:48,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:14:48,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:14:48,390 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:14:53,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:14:58,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:01,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:15:01,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:15:01,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:15:01,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:15:02,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:15:02,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:15:02,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:15:03,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:15:03,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:15:04,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:09,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:14,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:17,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:15:18,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:15:18,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:15:18,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:15:18,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:15:18,323 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:15:18,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:15:18,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:15:18,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:15:19,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:20,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:15:24,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:29,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:32,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:15:33,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:15:33,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:15:34,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:15:34,162 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:15:34,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:15:34,164 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:15:34,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:15:35,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:36,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:15:40,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:45,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:47,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:15:48,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:15:48,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:15:49,000 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:15:50,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:15:50,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:15:50,123 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:15:50,124 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:15:50,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:15:51,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:15:52,545 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:15:56,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:01,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:02,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:16:03,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:16:03,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:16:06,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:16:06,069 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:16:06,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:16:06,071 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:16:06,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:16:07,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:08,559 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:16:12,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:17,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:17,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:16:18,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:16:18,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:16:18,399 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:16:22,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:16:22,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:16:22,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:16:22,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:22,223 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:16:22,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:16:24,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:16:27,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:32,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:32,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:16:33,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:16:33,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:16:37,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:38,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:16:38,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:16:38,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:16:38,063 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:16:38,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:16:40,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:16:43,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:47,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:16:48,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:16:48,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:16:48,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:48,403 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:16:53,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:16:53,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:16:53,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:16:53,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:16:53,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:16:54,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:16:54,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:16:58,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:02,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:17:03,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:17:03,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:17:04,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:09,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:10,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:17:10,084 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:17:10,084 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:17:10,086 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:17:10,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:17:12,615 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:17:15,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:17,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:17:18,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:17:18,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:17:18,406 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:17:20,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:25,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:26,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:17:26,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:17:26,164 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:17:26,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:17:26,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:17:28,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:17:31,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:32,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:17:33,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:17:33,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:17:36,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:41,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:42,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:17:42,120 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:17:42,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:17:42,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:17:42,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:17:44,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:17:47,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:47,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:17:48,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:17:48,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:17:48,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:17:52,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:57,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:17:58,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:17:58,173 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:17:58,173 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:17:58,176 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:17:58,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:18:00,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:18:02,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:02,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:18:03,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:18:03,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:18:08,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:13,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:14,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:18:14,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:18:14,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:18:14,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:18:14,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:18:16,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:18:17,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:18:18,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:18:18,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:18:18,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:18,412 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:18:23,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:28,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:30,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:18:30,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:18:30,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:18:30,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:18:30,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:18:30,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:18:32,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:18:33,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:18:33,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:18:34,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:39,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:44,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:46,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:18:46,266 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:18:46,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:18:46,268 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:18:46,695 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:18:47,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:18:48,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:18:48,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:18:48,415 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:18:48,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:18:49,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:54,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:18:59,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:02,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:19:02,239 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:19:02,239 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:19:02,241 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:19:02,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:19:02,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:19:03,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:19:03,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:19:04,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:19:05,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:10,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:15,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:17,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:19:18,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:19:18,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:19:18,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:19:18,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:19:18,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:19:18,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:19:18,418 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:19:18,721 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:19:20,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:20,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:19:25,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:30,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:32,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:19:33,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:19:33,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:19:34,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:19:34,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:19:34,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:19:34,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:19:34,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:19:36,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:36,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:19:41,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:46,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:47,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:19:48,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:19:48,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:19:48,422 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:19:50,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:19:50,344 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:19:50,344 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:19:50,346 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:19:50,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:19:52,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:19:52,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:19:57,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:02,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:02,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:20:03,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:20:03,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:20:06,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:20:06,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:20:06,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:20:06,548 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:20:06,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:20:07,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:08,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:20:12,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:17,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:20:18,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:20:18,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:20:18,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:18,423 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:20:22,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:20:22,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:20:22,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:20:22,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:20:22,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:20:23,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:24,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:20:28,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:32,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:20:33,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:20:33,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:20:34,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:38,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:20:38,638 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:20:38,638 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:20:38,641 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:20:38,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:20:39,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:40,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:20:44,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:47,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:20:48,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:20:48,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:20:48,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:20:50,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:54,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:20:54,673 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:20:54,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:20:54,675 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:20:54,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:20:55,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:20:56,804 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:21:00,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:02,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:21:03,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:21:03,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:21:06,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:10,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:21:10,658 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:21:10,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:21:10,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:21:10,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:21:11,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:12,817 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:21:16,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:17,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:21:18,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:21:18,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:21:18,431 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:21:22,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:26,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:21:26,823 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:21:26,823 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:21:26,825 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:21:26,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:21:27,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:28,831 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:21:32,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:21:33,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:21:33,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:21:33,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:38,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:43,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:21:43,021 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:21:43,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:21:43,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:21:43,844 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:21:44,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:44,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:21:47,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:21:48,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:21:48,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:21:48,433 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:21:49,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:54,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:21:58,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:21:58,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:21:58,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:21:58,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:21:59,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:21:59,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:00,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:22:02,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:22:03,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:22:03,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:22:05,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:10,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:15,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:22:15,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:22:15,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:22:15,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:22:15,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:22:16,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:16,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:22:17,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:22:18,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:22:18,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:22:18,435 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:22:21,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:26,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:31,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:22:31,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:22:31,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:22:31,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:22:31,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:22:32,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:32,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:22:32,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:22:33,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:22:33,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:22:38,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:43,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:47,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:22:47,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:22:47,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:22:47,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:22:47,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:22:47,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:22:48,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:22:48,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:22:48,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:48,438 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:22:48,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:22:53,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:22:58,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:02,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:23:03,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:23:03,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:23:03,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:23:03,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:23:03,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:23:03,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:03,629 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:23:03,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:23:04,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:23:08,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:13,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:17,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:23:18,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:23:18,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:23:18,441 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:23:19,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:19,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:23:19,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:23:19,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:23:19,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:23:19,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:23:20,932 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:23:24,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:29,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:32,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:23:33,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:23:33,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:23:35,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:35,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:23:35,937 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:23:35,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:23:35,939 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:23:35,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:23:36,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:23:40,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:45,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:47,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:23:48,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:23:48,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:23:48,445 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:23:51,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:23:51,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:23:51,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:23:51,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:23:51,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:23:52,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:23:52,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:23:56,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:01,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:02,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:24:03,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:24:03,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:24:07,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:08,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:24:08,187 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:24:08,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:24:08,189 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:24:08,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:24:08,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:24:13,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:17,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:24:18,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:24:18,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:24:18,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:18,447 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:24:23,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:24,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:24:24,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:24:24,537 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:24:24,538 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:24:24,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:24:27,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:24:28,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:32,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:24:33,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:24:33,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:24:34,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:39,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:40,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:24:40,590 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:24:40,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:24:40,592 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:24:41,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:24:43,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:24:44,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:47,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:24:48,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:24:48,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:24:48,451 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:24:50,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:55,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:24:56,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:24:56,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:24:56,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:24:56,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:24:57,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:24:59,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:25:00,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:02,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:25:03,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:25:03,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:25:06,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:11,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:12,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:25:12,839 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:25:12,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:25:12,841 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:25:13,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:25:15,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:25:16,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:17,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:25:18,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:25:18,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:25:18,453 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:25:22,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:27,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:28,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:25:28,865 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:25:28,865 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:25:28,867 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:25:29,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:25:31,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:25:32,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:32,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:25:33,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:25:33,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:25:38,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:43,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:44,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:25:44,901 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:25:44,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:25:44,903 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:25:45,072 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:25:47,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:25:47,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:25:48,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:25:48,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:25:48,456 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:25:48,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:53,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:25:58,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:00,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:26:00,753 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:26:00,754 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:26:00,755 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:26:01,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:26:02,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:26:03,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:26:03,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:26:03,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:26:04,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:09,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:14,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:16,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:26:16,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:26:16,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:26:16,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:26:17,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:26:17,095 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:26:17,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:26:18,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:26:18,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:26:18,471 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:26:19,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:24,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:29,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:32,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:26:32,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:26:32,774 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:26:32,774 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:26:32,776 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:26:33,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:26:33,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:26:33,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:26:35,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:26:35,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:40,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:45,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:47,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:26:48,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:26:48,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:26:48,462 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:26:48,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:26:48,836 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:26:48,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:26:48,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:26:49,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:26:50,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:26:51,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:26:55,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:00,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:02,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:27:03,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:27:03,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:27:04,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:27:04,831 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:27:04,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:27:04,833 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:27:05,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:27:06,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:07,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:27:11,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:16,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:17,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:27:18,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:27:18,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:27:18,465 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:27:20,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:27:20,906 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:27:20,906 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:27:20,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:27:21,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:27:21,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:23,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:27:26,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:31,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:32,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:27:33,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:27:33,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:27:36,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:27:36,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:27:36,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:27:36,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:36,953 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:27:37,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:27:39,155 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:27:41,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:46,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:47,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:27:48,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:27:48,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:27:48,467 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:27:52,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:27:53,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:27:53,092 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:27:53,092 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:27:53,095 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:27:53,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:27:56,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:27:58,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:02,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:28:03,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:28:03,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:28:03,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:08,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:09,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:28:09,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:28:09,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:28:09,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:28:09,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:28:12,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:28:14,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:17,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:28:18,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:28:18,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:28:18,473 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:28:19,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:24,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:25,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:28:25,326 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:28:25,327 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:28:25,328 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:28:26,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:28:28,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:28:30,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:32,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:28:33,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:28:33,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:28:36,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:41,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:28:41,175 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:28:41,176 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:28:41,177 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:28:41,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:28:42,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:44,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:28:47,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:47,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:28:48,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:28:48,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:28:48,473 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:28:52,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:28:57,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:28:57,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:28:57,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:28:57,464 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:28:58,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:28:58,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:00,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:29:02,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:29:03,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:29:03,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:29:04,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:09,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:13,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:29:13,414 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:29:13,415 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:29:13,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:29:14,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:29:14,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:16,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:29:17,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:29:18,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:29:18,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:29:18,478 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:29:19,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:24,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:29,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:29:29,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:29:29,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:29:29,464 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:29:30,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:29:30,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:32,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:29:32,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:29:33,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:29:33,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:29:36,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:41,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:45,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:29:45,525 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:29:45,526 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:29:45,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:29:46,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:29:46,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:47,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:29:48,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:29:48,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:29:48,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:29:48,482 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:29:52,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:29:57,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:01,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:30:01,470 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:30:01,470 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:30:01,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:30:02,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:30:02,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:30:03,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:30:03,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:30:03,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:04,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:30:08,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:13,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:17,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:30:17,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:30:17,783 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:30:17,783 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:30:17,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:30:18,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:30:18,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:30:18,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:30:18,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:18,484 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:30:20,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:30:23,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:28,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:32,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:30:33,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:30:33,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:30:33,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:30:33,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:30:33,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:30:33,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:33,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:30:34,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:30:36,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:30:38,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:43,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:47,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:30:48,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:30:48,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:30:48,616 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:30:49,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:30:50,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:30:50,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:30:50,081 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:30:50,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:30:50,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:30:52,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:30:55,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:00,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:02,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:31:03,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:31:03,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:31:05,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:06,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:31:06,111 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:31:06,111 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:31:06,114 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:31:06,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:31:08,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:31:11,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:16,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:17,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:31:18,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:31:18,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:31:18,490 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:31:21,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:22,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:31:22,147 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:31:22,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:31:22,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:31:22,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:31:24,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:31:27,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:32,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:32,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:31:33,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:31:33,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:31:37,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:38,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:31:38,204 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:31:38,204 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:31:38,206 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:31:38,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:31:40,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:31:43,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:47,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:31:48,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:31:48,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:31:48,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:48,493 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:31:53,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:31:54,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:31:54,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:31:54,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:31:54,462 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:31:55,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:31:56,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:31:59,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:02,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:32:03,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:32:03,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:32:05,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:10,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:10,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:32:10,404 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:32:10,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:32:10,406 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:32:11,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:32:12,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:32:15,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:17,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:32:18,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:32:18,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:32:18,496 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:32:20,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:25,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:26,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:32:26,659 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:32:26,660 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:32:26,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:32:27,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:32:28,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:32:30,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:32,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:32:33,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:32:33,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:32:36,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:41,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:42,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:32:42,890 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:32:42,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:32:42,892 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:32:43,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:32:44,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:32:46,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:47,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:32:48,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:32:48,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:32:48,499 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:32:52,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:57,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:32:58,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:32:58,952 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:32:58,952 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:32:58,954 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:32:59,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:33:00,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:33:02,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:33:03,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:33:03,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:33:03,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:08,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:13,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:15,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:33:15,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:33:15,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:33:15,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:33:15,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:33:17,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:33:18,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:33:18,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:33:18,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:18,449 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:33:18,502 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:33:23,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:28,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:31,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:33:31,205 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:33:31,206 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:33:31,208 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:33:31,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:33:32,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:33:33,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:33:33,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:33:34,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:34,463 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:33:39,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:44,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:47,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:33:47,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:33:47,567 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:33:47,567 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:33:47,569 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:33:48,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:33:48,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:33:48,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:33:48,504 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:33:49,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:50,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:33:54,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:33:59,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:02,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:34:03,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:34:03,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:34:03,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:34:03,601 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:34:03,601 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:34:03,603 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:34:04,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:34:04,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:06,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:34:09,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:14,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:17,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:34:18,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:34:18,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:34:18,507 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:34:19,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:34:19,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:34:19,858 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:34:19,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:19,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:34:20,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:34:22,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:34:24,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:29,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:32,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:34:33,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:34:33,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:34:35,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:36,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:34:36,024 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:34:36,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:34:36,026 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:34:36,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:34:38,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:34:41,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:46,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:47,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:34:48,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:34:48,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:34:48,510 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:34:51,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:34:52,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:34:52,148 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:34:52,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:34:52,150 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:34:52,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:34:54,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:34:57,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:02,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:02,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:35:03,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:35:03,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:35:07,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:08,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:35:08,263 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:35:08,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:35:08,265 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:35:08,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:35:10,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:35:13,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:17,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:35:18,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:35:18,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:35:18,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:18,513 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:35:23,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:24,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:35:24,386 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:35:24,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:35:24,389 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:35:24,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:35:26,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:35:29,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:32,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:35:33,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:35:33,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:35:35,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:40,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:40,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:35:40,532 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:35:40,533 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:35:40,534 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:35:40,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:35:42,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:35:45,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:47,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:35:48,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:35:48,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:35:48,516 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:35:51,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:56,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:35:56,507 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:35:56,507 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:35:56,509 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:35:56,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:35:57,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:35:58,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:36:02,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:36:03,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:36:03,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:36:03,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:08,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:12,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:36:12,369 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:36:12,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:36:12,370 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:36:12,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:36:13,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:14,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:36:17,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:36:18,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:36:18,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:36:18,519 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:36:18,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:23,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:28,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:36:28,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:36:28,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:36:28,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:36:28,614 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:36:29,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:30,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:36:32,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:36:33,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:36:33,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:36:35,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:40,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:44,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:36:44,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:36:44,544 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:36:44,545 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:36:44,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:36:45,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:46,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:36:47,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:36:48,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:36:48,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:36:48,524 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:36:51,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:36:56,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:00,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:37:00,601 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:37:00,601 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:37:00,603 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:37:00,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:37:01,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:02,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:37:02,646 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:37:03,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:37:03,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:37:07,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:12,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:16,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:37:16,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:37:16,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:37:16,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:37:16,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:37:17,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:17,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:37:18,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:37:18,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:37:18,526 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:37:18,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:37:22,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:27,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:32,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:37:32,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:37:32,629 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:37:32,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:37:32,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:32,631 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:37:32,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:37:33,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:37:33,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:37:34,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:37:38,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:43,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:47,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:37:48,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:37:48,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:37:48,529 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:37:48,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:48,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:37:48,695 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:37:48,696 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:37:48,697 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:37:49,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:37:50,685 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:37:53,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:37:58,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:02,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:38:03,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:38:03,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:38:04,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:04,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:38:04,779 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:38:04,780 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:38:04,781 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:38:05,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:38:06,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:38:09,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:14,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:17,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:38:18,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:38:18,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:38:18,533 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:38:20,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:20,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:38:20,952 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:38:20,953 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:38:20,955 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:38:21,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:38:22,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:38:25,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:30,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:32,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:38:33,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:38:33,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:38:36,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:37,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:38:37,083 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:38:37,083 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:38:37,084 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:38:37,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:38:38,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:38:42,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:47,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:47,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:38:48,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:38:48,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:38:48,535 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:38:52,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:38:53,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:38:53,029 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:38:53,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:38:53,031 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:38:53,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:38:54,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:38:58,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:02,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:39:03,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:39:03,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:39:03,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:08,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:09,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:39:09,338 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:39:09,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:39:09,339 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:39:09,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:39:10,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:39:13,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:17,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:39:18,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:39:18,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:39:18,537 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:39:18,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:23,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:25,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:39:25,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:39:25,411 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:39:25,412 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:39:25,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:39:26,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:39:29,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:32,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:39:33,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:39:33,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:39:35,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:40,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:41,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:39:41,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:39:41,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:39:41,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:39:41,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:39:44,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:39:45,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:47,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:39:48,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:39:48,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:39:48,540 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:39:51,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:56,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:39:57,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:39:57,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:39:57,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:39:57,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:39:57,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:39:58,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:40:02,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:02,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:40:03,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:40:03,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:40:08,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:13,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:13,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:40:13,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:40:13,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:40:13,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:40:13,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:40:14,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:40:17,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:40:18,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:40:18,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:40:18,543 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:40:18,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:23,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:28,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:29,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:40:29,879 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:40:29,880 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:40:29,881 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:40:30,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:40:32,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:40:32,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:40:33,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:40:33,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:40:34,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:39,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:44,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:45,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:40:45,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:40:45,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:40:45,857 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:40:46,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:40:47,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:40:48,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:40:48,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:40:48,546 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:40:48,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:40:49,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:54,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:40:59,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:02,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:41:02,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:41:02,072 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:41:02,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:41:02,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:41:02,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:41:03,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:41:03,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:41:04,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:41:05,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:10,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:15,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:17,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:41:18,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:41:18,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:41:18,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:41:18,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:41:18,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:41:18,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:41:18,548 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:41:18,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:41:20,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:20,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:41:25,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:30,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:32,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:41:33,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:41:33,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:41:34,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:41:34,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:41:34,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:41:34,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:41:34,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:41:36,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:36,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:41:41,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:46,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:47,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:41:48,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:41:48,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:41:48,553 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:41:50,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:41:50,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:41:50,409 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:41:50,410 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:41:50,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:41:51,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:41:52,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:41:56,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:01,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:02,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:42:03,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:42:03,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:42:06,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:42:06,557 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:42:06,558 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:42:06,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:06,559 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:42:06,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:42:08,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:42:11,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:16,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:17,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:42:18,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:42:18,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:42:18,555 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:42:22,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:42:22,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:42:22,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:42:22,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:22,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:42:22,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:42:24,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:42:27,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:32,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:42:33,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:42:33,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:42:33,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:38,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:38,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:42:38,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:42:38,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:42:38,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:42:38,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:42:40,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:42:43,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:47,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:42:48,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:42:48,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:42:48,558 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:42:48,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:53,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:42:54,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:42:54,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:42:54,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:42:54,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:42:54,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:42:56,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:42:58,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:02,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:43:03,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:43:03,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:43:04,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:09,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:10,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:43:10,819 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:43:10,819 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:43:10,821 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:43:10,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:43:12,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:43:14,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:17,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:43:18,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:43:18,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:43:18,561 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:43:20,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:25,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:27,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:43:27,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:43:27,042 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:43:27,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:43:27,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:43:28,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:43:31,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:32,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:43:33,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:43:33,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:43:36,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:41,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:43,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:43:43,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:43:43,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:43:43,104 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:43:43,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:43:44,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:43:47,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:47,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:43:48,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:43:48,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:43:48,564 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:43:52,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:57,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:43:59,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:43:59,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:43:59,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:43:59,427 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:44:00,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:44:01,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:44:02,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:44:03,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:44:03,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:44:03,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:08,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:13,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:15,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:44:15,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:44:15,433 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:44:15,435 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:44:16,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:44:17,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:44:17,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:44:18,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:44:18,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:44:18,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:18,567 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:44:23,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:28,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:31,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:44:31,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:44:31,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:44:31,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:44:32,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:44:32,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:44:33,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:44:33,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:44:33,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:44:34,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:39,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:44,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:47,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:44:48,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:44:48,020 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:44:48,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:44:48,022 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:44:48,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:44:48,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:44:48,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:44:48,569 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:44:49,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:51,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:44:54,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:44:59,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:02,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:45:03,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:45:03,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:45:04,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:45:04,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:45:04,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:45:04,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:45:04,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:45:05,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:07,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:45:10,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:15,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:17,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:45:18,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:45:18,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:45:18,574 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:45:20,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:45:20,267 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:45:20,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:45:20,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:20,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:45:21,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:45:23,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:45:25,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:30,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:32,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:45:33,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:45:33,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:45:36,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:36,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:45:36,279 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:45:36,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:45:36,281 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:45:37,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:45:39,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:45:41,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:46,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:47,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:45:48,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:45:48,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:45:48,576 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:45:51,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:45:52,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:45:52,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:45:52,439 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:45:52,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:45:53,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:45:55,103 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:45:57,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:02,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:02,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:46:03,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:46:03,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:46:08,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:08,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:46:08,496 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:46:08,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:46:08,498 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:46:09,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:46:11,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:46:13,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:17,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:46:18,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:46:18,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:46:18,578 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:46:18,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:23,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:24,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:46:24,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:46:24,349 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:46:24,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:46:25,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:46:27,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:46:29,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:32,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:46:33,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:46:33,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:46:35,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:40,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:40,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:46:40,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:46:40,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:46:40,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:46:41,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:46:43,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:46:45,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:47,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:46:48,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:46:48,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:46:48,582 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:46:50,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:55,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:46:56,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:46:56,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:46:56,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:46:56,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:46:57,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:46:59,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:47:01,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:02,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:47:03,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:47:03,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:47:07,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:12,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:12,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:47:12,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:47:12,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:47:12,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:47:13,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:47:15,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:47:17,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:17,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:47:18,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:47:18,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:47:18,584 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:47:22,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:27,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:28,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:47:28,525 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:47:28,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:47:28,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:47:29,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:47:31,188 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:47:32,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:47:33,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:47:33,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:47:33,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:38,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:43,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:44,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:47:44,485 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:47:44,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:47:44,487 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:47:45,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:47:47,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:47:47,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:47:48,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:47:48,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:47:48,588 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:47:48,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:53,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:47:58,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:01,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:48:01,016 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:48:01,017 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:48:01,018 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:48:01,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:48:02,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:48:03,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:48:03,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:48:03,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:48:04,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:09,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:14,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:17,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:48:17,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:48:17,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:48:17,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:48:17,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:48:17,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:48:18,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:48:18,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:48:18,591 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:48:19,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:48:19,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:24,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:29,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:32,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:48:33,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:48:33,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:48:33,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:48:33,265 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:48:33,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:48:33,267 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:48:34,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:48:35,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:48:35,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:40,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:45,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:47,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:48:48,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:48:48,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:48:48,596 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:48:49,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:48:49,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:48:49,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:48:49,287 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:48:50,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:48:50,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:48:51,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:48:55,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:00,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:02,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:49:03,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:49:03,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:49:05,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:49:05,230 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:49:05,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:49:05,232 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:49:05,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:49:06,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:07,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:49:11,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:16,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:17,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:49:18,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:49:18,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:49:18,596 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:49:21,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:49:21,525 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:49:21,526 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:49:21,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:21,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:49:22,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:49:23,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:49:26,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:31,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:32,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:49:33,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:49:33,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:49:37,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:37,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:49:37,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:49:37,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:49:37,677 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:49:38,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:49:39,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:49:42,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:47,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:49:48,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:49:48,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:49:48,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:48,600 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:49:53,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:49:53,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:49:53,743 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:49:53,743 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:49:53,745 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:49:54,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:49:55,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:49:58,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:02,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:50:03,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:50:03,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:50:04,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:09,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:09,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:50:09,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:50:09,955 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:50:09,956 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:50:10,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:50:11,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:50:14,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:17,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:50:18,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:50:18,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:50:18,604 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:50:20,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:25,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:25,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:50:25,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:50:25,999 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:50:26,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:50:26,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:50:27,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:50:31,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:32,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:50:33,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:50:33,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:50:36,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:41,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:42,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:50:42,375 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:50:42,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:50:42,377 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:50:43,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:50:45,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:50:46,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:47,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:50:48,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:50:48,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:50:48,604 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:50:51,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:56,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:50:58,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:50:58,324 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:50:58,324 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:50:58,326 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:50:58,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:51:01,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:51:02,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:02,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:51:03,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:51:03,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:51:08,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:13,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:14,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:51:14,475 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:51:14,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:51:14,477 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:51:15,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:51:17,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:51:17,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:51:18,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:51:18,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:51:18,608 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:51:18,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:23,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:28,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:30,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:51:30,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:51:30,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:51:30,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:51:31,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:51:32,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:51:33,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:51:33,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:51:33,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:51:34,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:39,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:44,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:46,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:51:46,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:51:46,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:51:46,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:51:47,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:51:47,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:51:48,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:51:48,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:51:48,612 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:51:49,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:51:49,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:54,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:51:59,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:02,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:52:02,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:52:02,516 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:52:02,516 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:52:02,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:52:03,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:52:03,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:52:03,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:52:05,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:05,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:52:10,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:15,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:17,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:52:18,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:52:18,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:52:18,614 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:52:18,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:52:18,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:52:18,665 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:52:18,666 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:52:19,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:52:20,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:21,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:52:25,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:30,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:32,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:52:33,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:52:33,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:52:34,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:52:34,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:52:34,613 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:52:34,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:52:35,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:52:36,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:37,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:52:41,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:46,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:47,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:52:48,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:52:48,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:52:48,617 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:52:50,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:52:50,764 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:52:50,765 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:52:50,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:52:51,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:52:51,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:52:53,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:52:56,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:01,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:02,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:53:03,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:53:03,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:53:07,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:53:07,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:53:07,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:53:07,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:07,049 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:53:07,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:53:09,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:53:12,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:17,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:17,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:53:18,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:53:18,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:53:18,620 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:53:22,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:23,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:53:23,098 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:53:23,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:53:23,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:53:23,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:53:25,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:53:28,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:32,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:53:33,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:53:33,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:53:33,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:38,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:39,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:53:39,335 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:53:39,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:53:39,338 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:53:39,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:53:41,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:53:43,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:47,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:53:48,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:53:48,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:53:48,624 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:53:48,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:53,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:53:55,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:53:55,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:53:55,305 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:53:55,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:53:55,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:53:57,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:53:59,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:02,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:54:03,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:54:03,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:54:05,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:10,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:11,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:54:11,648 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:54:11,648 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:54:11,650 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:54:12,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:54:13,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:54:15,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:17,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:54:18,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:54:18,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:54:18,626 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:54:21,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:26,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:27,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:54:27,667 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:54:27,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:54:27,669 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:54:28,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:54:29,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:54:31,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:32,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:54:33,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:54:33,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:54:37,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:42,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:43,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:54:43,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:54:43,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:54:43,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:54:44,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:54:45,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:54:47,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:47,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:54:48,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:54:48,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:54:48,630 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:54:52,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:54:57,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:00,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:55:00,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:55:00,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:55:00,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:55:00,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:55:01,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:55:02,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:55:03,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:55:03,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:55:03,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:08,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:13,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:16,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:55:16,298 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:55:16,299 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:55:16,301 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:55:16,601 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:55:17,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:55:17,602 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:55:18,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:55:18,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:55:18,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:18,634 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:55:23,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:28,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:32,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:55:32,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:55:32,506 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:55:32,506 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:55:32,508 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:55:32,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:55:33,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:55:33,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:55:33,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:55:34,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:39,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:44,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:47,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:55:48,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:55:48,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:55:48,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:55:48,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:55:48,549 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:55:48,552 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:55:48,636 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:55:48,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:55:49,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:49,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:55:54,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:55:59,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:02,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:56:03,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:56:03,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:56:04,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:56:04,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:56:04,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:56:04,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:04,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:56:05,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:56:05,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:56:09,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:14,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:17,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:56:18,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:56:18,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:56:18,637 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:56:20,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:20,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:56:20,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:56:20,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:56:20,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:56:21,662 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:56:21,662 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:56:25,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:30,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:32,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:56:33,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:56:33,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:56:36,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:36,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:56:36,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:56:36,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:56:36,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:56:36,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:56:37,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:56:41,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:46,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:47,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:56:48,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:56:48,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:56:48,641 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:56:51,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:56:52,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:56:52,441 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:56:52,441 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:56:52,443 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:56:52,687 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:56:53,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:56:57,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:02,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:02,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:57:03,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:57:03,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:57:08,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:08,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:57:08,717 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:57:08,718 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:57:08,719 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:57:09,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:57:09,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:57:13,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:17,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:57:18,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:57:18,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:57:18,645 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:57:19,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:24,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:24,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:57:24,665 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:57:24,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:57:24,667 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:57:24,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:57:25,721 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:57:29,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:32,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:57:33,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:57:33,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:57:35,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:40,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:40,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:57:40,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:57:40,626 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:57:40,627 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:57:40,734 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:57:41,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:57:45,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:47,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:57:48,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:57:48,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:57:48,647 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:57:50,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:55,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:57:56,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:57:56,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:57:56,802 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:57:56,803 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:57:57,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:57:57,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:58:00,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:02,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:58:03,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:58:03,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:58:06,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:11,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:12,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:58:12,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:58:12,669 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:58:12,671 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:58:12,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:58:13,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:58:16,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:17,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:58:18,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:58:18,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:58:18,650 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:58:22,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:27,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:28,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:58:28,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:58:28,744 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:58:28,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:58:28,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:58:29,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:58:32,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:58:33,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:58:33,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:58:33,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:38,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:43,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:44,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:58:44,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:58:44,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:58:44,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:58:45,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:58:45,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:58:47,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:58:48,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:58:48,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:58:48,654 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:58:48,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:53,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:58:58,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:01,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:59:01,154 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:59:01,154 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:59:01,156 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:59:01,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:59:01,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:59:02,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:59:03,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:59:03,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:59:04,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:09,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:14,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:17,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:59:17,231 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:59:17,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:59:17,233 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:59:17,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:59:17,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:59:18,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:59:18,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:59:18,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:59:19,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:19,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:59:24,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:29,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:32,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:59:33,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:59:33,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:59:33,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:59:33,340 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:59:33,340 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:59:33,342 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:59:33,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:59:35,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:35,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:59:40,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:45,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:47,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 00:59:48,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 00:59:48,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 00:59:48,659 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 00:59:49,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 00:59:49,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 00:59:49,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 00:59:49,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 00:59:49,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 00:59:50,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 00:59:51,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 00:59:55,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:00,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:02,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:00:03,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:00:03,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:00:05,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:00:05,509 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:00:05,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:00:05,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:05,511 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:00:05,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:00:07,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:00:10,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:15,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:17,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:00:18,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:00:18,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:00:18,662 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:00:20,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:21,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:00:21,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:00:21,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:00:21,678 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:00:21,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:00:23,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:00:25,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:30,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:32,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:00:33,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:00:33,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:00:36,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:37,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:00:37,690 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:00:37,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:00:37,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:00:37,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:00:39,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:00:41,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:46,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:47,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:00:48,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:00:48,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:00:48,664 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:00:52,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:00:53,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:00:53,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:00:53,923 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:00:53,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:00:54,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:00:55,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:00:57,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:02,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:01:03,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:01:03,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:01:03,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:08,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:09,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:01:09,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:01:09,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:01:09,871 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:01:09,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:01:11,910 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:01:13,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:17,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:01:18,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:01:18,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:01:18,667 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:01:19,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:24,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:26,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:01:26,246 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:01:26,246 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:01:26,248 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:01:26,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:01:27,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:01:30,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:32,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:01:33,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:01:33,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:01:35,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:40,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:42,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:01:42,199 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:01:42,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:01:42,201 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:01:42,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:01:43,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:01:46,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:47,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:01:48,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:01:48,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:01:48,669 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:01:51,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:56,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:01:58,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:01:58,137 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:01:58,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:01:58,139 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:01:58,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:01:59,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:02:02,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:02,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:02:03,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:02:03,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:02:07,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:12,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:14,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:02:14,283 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:02:14,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:02:14,285 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:02:14,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:02:15,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:02:17,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:17,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:02:18,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:02:18,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:02:18,674 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:02:22,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:27,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:30,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:02:30,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:02:30,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:02:30,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:02:30,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:02:31,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:02:32,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:02:33,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:02:33,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:02:33,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:38,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:43,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:46,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:02:46,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:02:46,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:02:46,412 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:02:46,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:02:47,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:02:47,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:02:48,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:02:48,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:02:48,676 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:02:48,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:53,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:02:58,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:02,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:03:02,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:03:02,570 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:03:02,570 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:03:02,572 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:03:03,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:03:03,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:03:03,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:03:04,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:03:04,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:09,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:14,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:17,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:03:18,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:03:18,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:03:18,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:03:18,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:03:18,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:03:18,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:03:18,678 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:03:19,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:03:19,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:20,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:03:24,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:29,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:32,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:03:33,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:03:33,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:03:34,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:03:34,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:03:34,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:03:34,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:34,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:03:35,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:03:36,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:03:39,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:44,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:47,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:03:48,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:03:48,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:03:48,682 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:03:50,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:03:50,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:03:50,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:03:50,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:03:50,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:03:51,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:03:52,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:03:55,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:00,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:02,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:04:03,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:04:03,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:04:06,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:07,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:04:07,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:04:07,013 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:04:07,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:04:07,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:04:08,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:04:12,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:17,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:17,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:04:18,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:04:18,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:04:18,685 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:04:22,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:23,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:04:23,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:04:23,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:04:23,159 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:04:24,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:04:24,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:04:28,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:32,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:04:33,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:04:33,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:04:33,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:38,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:39,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:04:39,190 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:04:39,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:04:39,191 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:04:40,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:04:40,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:04:44,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:47,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:04:48,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:04:48,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:04:48,688 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:04:49,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:54,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:04:55,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:04:55,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:04:55,303 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:04:55,304 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:04:56,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:04:58,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:05:00,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:02,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:05:03,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:05:03,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:05:06,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:11,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:11,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:05:11,344 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:05:11,344 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:05:11,346 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:05:12,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:05:14,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:05:16,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:17,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:05:18,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:05:18,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:05:18,692 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:05:21,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:26,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:27,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:05:27,393 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:05:27,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:05:27,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:05:28,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:05:30,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:05:32,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:32,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:05:33,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:05:33,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:05:38,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:43,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:43,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:05:43,431 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:05:43,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:05:43,433 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:05:44,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:05:46,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:05:47,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:05:48,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:05:48,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:05:48,695 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:05:48,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:53,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:58,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:05:59,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:05:59,404 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:05:59,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:05:59,406 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:06:00,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:06:02,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:06:02,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:06:03,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:06:03,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:06:04,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:09,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:14,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:15,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:06:15,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:06:15,433 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:06:15,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:06:16,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:06:17,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:06:18,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:06:18,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:06:18,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:06:18,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:06:19,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:24,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:29,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:31,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:06:31,753 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:06:31,753 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:06:31,755 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:06:32,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:06:32,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:06:33,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:06:33,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:06:34,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:06:35,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:40,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:45,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:47,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:06:47,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:06:47,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:06:47,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:06:47,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:06:48,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:06:48,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:06:48,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:06:48,700 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:06:50,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:06:50,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:06:55,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:00,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:02,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:07:03,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:07:03,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:07:03,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:07:03,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:07:03,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:07:03,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:07:04,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:07:06,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:06,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:11,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:16,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:17,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:07:18,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:07:18,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:07:18,704 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:07:19,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:07:19,672 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:07:19,673 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:07:19,674 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:07:20,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:07:21,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:22,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:24,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:27,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:28,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:30,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:31,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:32,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:32,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:07:33,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:07:33,123 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:07:36,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:38,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:38,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:39,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:07:42,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:07:43,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:47,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:07:48,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:07:48,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:07:48,705 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:07:49,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:54,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:07:56,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:07:56,047 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:07:56,048 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:07:56,050 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:07:56,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:07:58,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:08:00,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:02,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:08:03,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:08:03,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:08:05,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:10,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:12,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:08:12,499 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:08:12,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:08:12,501 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:08:13,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:08:14,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:08:15,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:17,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:08:18,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:08:18,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:08:18,743 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:08:20,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:25,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:28,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:08:28,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:08:28,537 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:08:28,538 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:08:29,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:08:30,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:08:31,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:32,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:08:33,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:08:33,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:08:37,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:42,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:44,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:08:44,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:08:44,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:08:44,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:08:45,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:08:46,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:08:47,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:47,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:08:48,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:08:48,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:08:48,746 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:08:52,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:08:57,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:01,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:09:01,152 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:09:01,152 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:09:01,153 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:09:01,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:09:02,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:09:02,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:09:03,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:09:03,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:09:03,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:08,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:13,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:17,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:09:17,103 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:09:17,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:09:17,105 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:09:17,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:09:17,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:09:18,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:09:18,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:09:18,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:09:18,750 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:09:18,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:23,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:28,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:32,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:09:33,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:09:33,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:09:33,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:09:33,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:09:33,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:09:33,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:09:33,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:09:34,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:34,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:09:39,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:44,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:47,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:09:48,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:09:48,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:09:48,753 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:09:49,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:09:49,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:09:49,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:09:49,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:09:49,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:09:50,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:09:50,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:09:55,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:00,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:02,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:10:03,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:10:03,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:10:05,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:10:05,019 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:10:05,019 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:10:05,021 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:10:05,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:10:06,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:06,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:10:11,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:16,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:17,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:10:18,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:10:18,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:10:18,756 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:10:21,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:10:21,083 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:10:21,084 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:10:21,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:21,086 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:10:21,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:10:22,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:10:26,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:31,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:32,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:10:33,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:10:33,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:10:36,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:36,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:10:36,947 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:10:36,947 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:10:36,949 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:10:37,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:10:38,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:10:41,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:46,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:47,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:10:48,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:10:48,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:10:48,760 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:10:52,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:10:52,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:10:52,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:10:52,999 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:10:53,000 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:10:53,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:10:54,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:10:58,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:02,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:11:03,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:11:03,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:11:03,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:08,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:08,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:11:08,860 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:11:08,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:11:08,862 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:11:09,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:11:10,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:11:13,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:17,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:11:18,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:11:18,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:11:18,762 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:11:19,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:24,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:25,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:11:25,159 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:11:25,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:11:25,161 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:11:25,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:11:26,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:11:30,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:32,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:11:33,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:11:33,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:11:35,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:40,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:41,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:11:41,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:11:41,123 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:11:41,124 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:11:41,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:11:42,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:11:46,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:47,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:11:48,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:11:48,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:11:48,765 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:11:51,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:56,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:11:57,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:11:57,185 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:11:57,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:11:57,186 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:11:57,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:11:58,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:12:02,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:02,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:12:03,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:12:03,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:12:07,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:12,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:13,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:12:13,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:12:13,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:12:13,148 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:12:13,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:12:14,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:12:17,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:17,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:12:18,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:12:18,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:12:18,769 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:12:22,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:27,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:29,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:12:29,195 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:12:29,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:12:29,197 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:12:29,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:12:30,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:12:32,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:12:33,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:12:33,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:12:33,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:38,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:43,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:45,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:12:45,340 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:12:45,340 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:12:45,342 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:12:45,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:12:46,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:12:47,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:12:48,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:12:48,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:12:48,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:48,771 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:12:53,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:12:58,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:01,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:13:01,498 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:13:01,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:13:01,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:13:01,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:13:02,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:13:02,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:13:03,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:13:03,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:13:04,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:09,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:14,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:17,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:13:17,459 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:13:17,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:13:17,461 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:13:17,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:13:17,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:13:18,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:13:18,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:13:18,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:13:18,773 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:13:19,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:24,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:29,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:32,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:13:33,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:13:33,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:13:33,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:13:33,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:13:33,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:13:33,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:13:33,538 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:13:34,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:13:35,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:40,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:45,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:47,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:13:48,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:13:48,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:13:48,776 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:13:49,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:13:49,457 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:13:49,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:13:49,459 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:13:49,552 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:13:50,552 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:13:51,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:13:56,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:01,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:02,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:14:03,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:14:03,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:14:05,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:14:05,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:14:05,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:14:05,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:14:06,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:14:06,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:14:06,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:11,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:16,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:17,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:14:18,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:14:18,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:14:18,781 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:14:21,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:14:21,621 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:14:21,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:14:21,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:21,623 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:14:22,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:14:22,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:14:26,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:31,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:32,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:14:33,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:14:33,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:14:37,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:37,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:14:37,772 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:14:37,772 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:14:37,774 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:14:38,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:14:38,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:14:42,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:47,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:14:48,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:14:48,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:14:48,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:48,783 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:14:53,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:14:53,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:14:53,816 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:14:53,816 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:14:53,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:14:54,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:14:54,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:14:58,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:02,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:15:03,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:15:03,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:15:04,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:09,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:09,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:15:09,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:15:09,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:15:09,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:15:10,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:15:10,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:15:14,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:17,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:15:18,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:15:18,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:15:18,786 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:15:20,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:25,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:26,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:15:26,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:15:26,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:15:26,138 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:15:26,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:15:26,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:15:31,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:32,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:15:33,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:15:33,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:15:36,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:41,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:42,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:15:42,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:15:42,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:15:42,253 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:15:42,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:15:42,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:15:47,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:47,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:15:48,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:15:48,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:15:48,790 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:15:52,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:57,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:15:58,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:15:58,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:15:58,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:15:58,400 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:15:58,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:15:58,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:16:02,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:16:03,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:16:03,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:16:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:08,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:13,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:14,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:16:14,530 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:16:14,530 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:16:14,532 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:16:14,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:16:16,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:16:17,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:16:18,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:16:18,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:16:18,793 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:16:18,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:23,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:28,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:30,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:16:30,886 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:16:30,886 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:16:30,888 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:16:31,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:16:32,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:16:32,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:16:33,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:16:33,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:16:34,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:39,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:44,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:46,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:16:46,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:16:46,851 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:16:46,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:16:47,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:16:47,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:16:48,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:16:48,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:16:48,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:16:48,795 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:16:49,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:54,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:16:59,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:02,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:17:03,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:17:03,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:17:03,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:17:03,364 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:17:03,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:17:03,366 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:17:03,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:17:04,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:17:05,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:10,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:15,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:17,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:17:18,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:17:18,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:17:18,798 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:17:19,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:17:19,326 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:17:19,326 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:17:19,328 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:17:19,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:17:20,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:17:21,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:26,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:31,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:32,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:17:33,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:17:33,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:17:35,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:17:35,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:17:35,735 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:17:35,736 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:17:35,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:17:36,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:36,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:17:41,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:46,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:47,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:17:48,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:17:48,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:17:48,801 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:17:51,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:17:51,767 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:17:51,768 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:17:51,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:17:51,769 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:17:51,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:17:52,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:17:56,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:01,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:02,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:18:03,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:18:03,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:18:07,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:08,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:18:08,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:18:08,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:18:08,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:18:08,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:18:08,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:18:13,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:17,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:18:18,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:18:18,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:18:18,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:18,804 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:18:23,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:24,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:18:24,453 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:18:24,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:18:24,455 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:18:24,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:18:24,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:18:29,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:32,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:18:33,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:18:33,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:18:35,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:40,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:40,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:18:40,500 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:18:40,501 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:18:40,503 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:18:40,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:18:40,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:18:45,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:47,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:18:48,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:18:48,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:18:48,806 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:18:50,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:55,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:18:56,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:18:56,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:18:56,722 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:18:56,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:18:56,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:18:58,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:19:01,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:02,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:19:03,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:19:03,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:19:07,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:12,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:12,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:19:12,878 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:19:12,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:19:12,880 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:19:13,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:19:14,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:19:17,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:17,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:19:18,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:19:18,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:19:18,811 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:19:22,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:27,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:28,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:19:28,808 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:19:28,809 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:19:28,810 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:19:28,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:19:30,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:19:32,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:19:33,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:19:33,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:19:33,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:38,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:43,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:44,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:19:44,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:19:44,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:19:44,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:19:44,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:19:46,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:19:47,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:19:48,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:19:48,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:19:48,813 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:19:48,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:53,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:19:58,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:00,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:20:00,876 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:20:00,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:20:00,879 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:20:00,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:20:02,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:20:02,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:20:03,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:20:03,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:20:04,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:09,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:14,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:17,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:20:17,100 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:20:17,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:20:17,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:20:17,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:20:17,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:20:18,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:20:18,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:20:18,816 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:20:18,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:20:19,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:24,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:29,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:32,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:20:33,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:20:33,050 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:20:33,050 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:20:33,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:20:33,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:20:33,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:20:33,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:20:34,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:20:35,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:40,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:45,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:47,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:20:48,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:20:48,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:20:48,819 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:20:48,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:20:48,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:20:48,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:20:48,922 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:20:48,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:20:50,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:20:50,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:20:55,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:00,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:02,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:21:03,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:21:03,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:21:04,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:21:04,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:21:04,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:21:04,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:21:05,938 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:21:05,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:06,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:21:10,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:15,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:17,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:21:18,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:21:18,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:21:18,822 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:21:20,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:21:20,842 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:21:20,843 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:21:20,844 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:21:20,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:21:21,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:22,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:21:26,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:31,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:32,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:21:33,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:21:33,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:21:37,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:21:37,100 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:21:37,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:21:37,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:37,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:21:37,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:21:38,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:21:42,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:47,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:47,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:21:48,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:21:48,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:21:48,825 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:21:52,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:21:53,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:21:53,053 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:21:53,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:21:53,055 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:21:53,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:21:54,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:21:58,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:02,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:22:03,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:22:03,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:22:03,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:08,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:09,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:22:09,014 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:22:09,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:22:09,016 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:22:09,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:22:10,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:22:14,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:17,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:22:18,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:22:18,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:22:18,828 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:22:19,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:24,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:25,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:22:25,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:22:25,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:22:25,077 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:22:26,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:22:27,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:22:30,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:32,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:22:33,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:22:33,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:22:35,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:40,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:41,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:22:41,259 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:22:41,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:22:41,261 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:22:42,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:22:43,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:22:46,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:47,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:22:48,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:22:48,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:22:48,833 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:22:51,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:56,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:22:57,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:22:57,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:22:57,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:22:57,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:22:58,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:22:59,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:23:02,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:02,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:23:03,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:23:03,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:23:08,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:13,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:13,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:23:13,458 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:23:13,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:23:13,460 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:23:14,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:23:15,051 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:23:17,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:23:18,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:23:18,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:23:18,834 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:23:18,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:23,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:28,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:29,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:23:29,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:23:29,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:23:29,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:23:30,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:23:31,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:23:32,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:23:33,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:23:33,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:23:34,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:39,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:44,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:45,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:23:45,489 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:23:45,489 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:23:45,491 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:23:46,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:23:47,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:23:47,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:23:48,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:23:48,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:23:48,838 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:23:49,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:54,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:23:59,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:01,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:24:01,548 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:24:01,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:24:01,550 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:24:02,093 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:24:02,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:24:03,093 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:24:03,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:24:03,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:24:05,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:10,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:15,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:17,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:24:17,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:24:17,708 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:24:17,708 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:24:17,709 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:24:18,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:24:18,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:24:18,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:24:18,842 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:24:19,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:24:20,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:25,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:30,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:32,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:24:33,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:24:33,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:24:33,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:24:33,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:24:33,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:24:33,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:24:34,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:24:35,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:24:36,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:41,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:46,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:47,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:24:48,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:24:48,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:24:48,842 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:24:49,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:24:49,716 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:24:49,716 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:24:49,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:24:50,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:24:51,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:24:51,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:24:56,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:01,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:02,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:25:03,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:25:03,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:25:05,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:25:05,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:25:05,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:25:05,863 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:25:06,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:25:06,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:07,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:25:11,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:16,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:17,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:25:18,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:25:18,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:25:18,846 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:25:21,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:25:21,903 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:25:21,903 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:25:21,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:21,905 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:25:22,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:25:23,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:25:26,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:31,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:32,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:25:33,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:25:33,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:25:37,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:38,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:25:38,175 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:25:38,176 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:25:38,177 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:25:38,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:25:39,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:25:43,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:47,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:25:48,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:25:48,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:25:48,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:48,881 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:25:53,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:25:54,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:25:54,387 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:25:54,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:25:54,389 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:25:55,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:25:55,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:25:59,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:02,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:26:03,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:26:03,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:26:05,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:10,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:10,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:26:10,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:26:10,422 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:26:10,423 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:26:11,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:26:11,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:26:15,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:17,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:26:18,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:26:18,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:26:18,884 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:26:20,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:25,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:26,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:26:26,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:26:26,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:26:26,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:26:27,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:26:27,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:26:31,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:32,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:26:33,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:26:33,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:26:37,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:42,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:42,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:26:42,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:26:42,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:26:42,871 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:26:43,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:26:43,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:26:47,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:47,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:26:48,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:26:48,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:26:48,887 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:26:52,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:57,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:26:58,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:26:58,839 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:26:58,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:26:58,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:26:59,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:26:59,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:27:02,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:27:03,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:27:03,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:27:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:08,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:13,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:15,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:27:15,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:27:15,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:27:15,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:27:15,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:27:15,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:27:17,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:27:18,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:27:18,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:27:18,890 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:27:18,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:23,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:28,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:31,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:27:31,233 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:27:31,233 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:27:31,235 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:27:31,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:27:31,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:27:32,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:27:33,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:27:33,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:27:34,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:39,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:44,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:27:47,457 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:27:47,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:27:47,459 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:27:47,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:27:48,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:27:48,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:27:48,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:27:49,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:27:52,876 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:27:52,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:27:57,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:02,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:28:03,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:28:03,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:28:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:03,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:28:03,520 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:28:03,520 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:28:03,522 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:28:04,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:28:06,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:28:08,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:13,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:17,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:28:18,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:28:18,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:28:18,896 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:28:18,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:19,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:28:19,847 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:28:19,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:28:19,850 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:28:20,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:28:22,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:28:24,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:29,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:32,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:28:33,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:28:33,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:28:35,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:35,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:28:35,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:28:35,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:28:35,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:28:36,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:28:38,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:28:40,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:45,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:47,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:28:48,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:28:48,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:28:48,901 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:28:51,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:28:52,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:28:52,128 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:28:52,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:28:52,131 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:28:52,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:28:54,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:28:57,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:02,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:02,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:29:03,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:29:03,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:29:07,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:08,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:29:08,338 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:29:08,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:29:08,340 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:29:08,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:29:10,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:29:13,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:17,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:29:18,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:29:18,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:29:18,905 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:29:18,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:23,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:24,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:29:24,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:29:24,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:29:24,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:29:25,381 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:29:26,381 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:29:29,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:32,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:29:33,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:29:33,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:29:35,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:40,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:40,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:29:40,494 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:29:40,495 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:29:40,496 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:29:41,396 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:29:42,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:29:45,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:47,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:29:48,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:29:48,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:29:48,908 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:29:50,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:55,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:29:56,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:29:56,573 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:29:56,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:29:56,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:29:57,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:29:58,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:30:01,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:02,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:30:03,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:30:03,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:30:07,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:12,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:12,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:30:12,687 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:30:12,687 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:30:12,688 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:30:13,424 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:30:14,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:30:17,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:17,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:30:18,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:30:18,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:30:18,910 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:30:22,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:27,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:28,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:30:28,734 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:30:28,735 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:30:28,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:30:29,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:30:30,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:30:32,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:30:33,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:30:33,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:30:33,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:38,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:43,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:44,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:30:44,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:30:44,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:30:44,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:30:45,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:30:46,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:30:47,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:30:48,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:30:48,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:30:48,913 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:30:48,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:53,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:30:58,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:00,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:31:00,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:31:00,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:31:00,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:31:01,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:31:02,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:31:02,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:31:03,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:31:03,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:31:04,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:09,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:14,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:16,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:31:16,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:31:16,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:31:16,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:31:17,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:31:17,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:31:18,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:31:18,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:31:18,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:31:19,384 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:31:19,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:24,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:29,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:32,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:31:32,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:31:32,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:31:32,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:31:32,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:31:33,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:31:33,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:31:33,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:31:34,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:31:35,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:40,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:45,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:47,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:31:48,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:31:48,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:31:48,920 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:31:48,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:31:48,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:31:48,997 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:31:48,998 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:31:49,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:31:50,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:31:50,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:31:56,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:01,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:02,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:32:03,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:32:03,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:32:04,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:32:04,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:32:04,954 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:32:04,956 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:32:05,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:32:06,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:32:06,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:11,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:16,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:17,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:32:18,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:32:18,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:32:18,923 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:32:21,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:32:21,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:32:21,114 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:32:21,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:32:21,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:32:22,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:22,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:32:27,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:32,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:32,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:32:33,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:32:33,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:32:36,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:32:36,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:32:36,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:32:36,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:32:37,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:32:37,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:38,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:32:42,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:47,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:32:48,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:32:48,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:32:48,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:48,927 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:32:53,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:32:53,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:32:53,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:32:53,034 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:32:53,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:32:54,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:32:54,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:32:59,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:02,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:33:03,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:33:03,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:33:04,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:09,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:33:09,283 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:33:09,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:33:09,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:33:09,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:33:10,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:10,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:33:15,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:17,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:33:18,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:33:18,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:33:18,929 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:33:20,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:25,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:33:25,341 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:33:25,341 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:33:25,343 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:33:25,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:33:26,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:26,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:33:31,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:32,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:33:33,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:33:33,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:33:36,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:41,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:41,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:33:41,492 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:33:41,493 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:33:41,495 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:33:41,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:33:42,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:33:46,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:47,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:33:48,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:33:48,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:33:52,846 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:33:52,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:57,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:33:57,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:33:57,534 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:33:57,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:33:57,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:33:58,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:33:58,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:34:02,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:34:03,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:34:03,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:34:04,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:09,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:13,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:34:13,480 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:34:13,481 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:34:13,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:34:13,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:34:14,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:14,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:34:17,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:34:18,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:34:18,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:34:18,935 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:34:19,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:24,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:29,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:34:29,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:34:29,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:34:29,700 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:34:30,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:34:30,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:34:30,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:32,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:34:33,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:34:33,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:34:36,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:41,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:45,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:34:45,546 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:34:45,546 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:34:45,549 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:34:45,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:34:46,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:46,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:34:47,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:34:48,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:34:48,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:34:48,938 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:34:51,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:34:56,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:01,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:35:01,686 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:35:01,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:35:01,687 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:35:02,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:02,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:35:02,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:35:02,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:35:03,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:35:03,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:35:08,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:13,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:17,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:35:17,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:35:17,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:35:17,956 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:35:17,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:35:18,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:35:18,105 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:35:18,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:35:18,941 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:35:18,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:20,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:35:23,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:28,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:32,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:35:33,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:35:33,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:35:34,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:35:34,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:35:34,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:35:34,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:34,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:35:34,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:35:36,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:35:39,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:44,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:47,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:35:48,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:35:48,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:35:48,943 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:35:49,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:35:50,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:35:50,149 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:35:50,149 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:35:50,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:35:50,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:35:52,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:35:55,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:00,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:02,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:36:03,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:36:03,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:36:05,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:06,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:36:06,203 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:36:06,204 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:36:06,205 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:36:06,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:36:08,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:36:11,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:16,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:17,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:36:18,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:36:18,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:36:18,949 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:36:21,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:22,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:36:22,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:36:22,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:36:22,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:36:22,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:36:24,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:36:27,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:32,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:32,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:36:33,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:36:33,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:36:37,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:38,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:36:38,419 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:36:38,419 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:36:38,421 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:36:38,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:36:40,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:36:42,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:47,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:47,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:36:48,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:36:48,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:36:48,950 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:36:52,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:36:54,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:36:54,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:36:54,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:36:54,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:36:54,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:36:56,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:36:58,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:02,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:37:03,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:37:03,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:37:04,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:09,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:10,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:37:10,725 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:37:10,725 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:37:10,727 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:37:10,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:37:12,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:37:14,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:17,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:37:18,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:37:18,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:37:18,953 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:37:19,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:24,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:26,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:37:26,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:37:26,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:37:26,981 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:37:27,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:37:28,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:37:29,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:32,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:37:33,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:37:33,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:37:35,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:40,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:43,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:37:43,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:37:43,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:37:43,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:37:43,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:37:44,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:37:46,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:47,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:37:48,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:37:48,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:37:48,956 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:37:51,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:56,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:37:59,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:37:59,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:37:59,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:37:59,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:37:59,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:38:00,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:38:02,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:02,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:38:03,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:38:03,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:38:08,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:13,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:15,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:38:15,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:38:15,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:38:15,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:38:15,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:38:16,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:38:17,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:38:18,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:38:18,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:38:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:18,959 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:38:23,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:28,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:31,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:38:31,962 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:38:31,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:38:31,964 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:38:32,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:38:32,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:38:32,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:38:33,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:38:33,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:38:34,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:39,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:44,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:47,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:38:48,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:38:48,045 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:38:48,045 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:38:48,047 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:38:48,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:38:48,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:38:48,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:38:48,962 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:38:49,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:50,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:38:54,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:38:59,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:02,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:39:03,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:39:03,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:39:04,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:39:04,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:39:04,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:39:04,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:39:04,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:39:05,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:06,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:39:10,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:15,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:17,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:39:18,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:39:18,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:39:18,965 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:39:20,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:39:20,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:39:20,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:39:20,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:20,473 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:39:20,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:39:22,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:39:25,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:30,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:32,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:39:33,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:39:33,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:39:36,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:36,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:39:36,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:39:36,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:39:36,485 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:39:36,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:39:38,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:39:41,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:46,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:47,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:39:48,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:39:48,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:39:48,969 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:39:51,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:39:52,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:39:52,659 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:39:52,660 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:39:52,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:39:52,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:39:54,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:39:57,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:02,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:40:03,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:40:03,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:40:03,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:08,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:08,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:40:08,513 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:40:08,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:40:08,515 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:40:08,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:40:10,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:40:13,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:17,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:40:18,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:40:18,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:40:18,971 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:40:18,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:23,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:24,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:40:24,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:40:24,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:40:24,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:40:24,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:40:26,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:40:29,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:32,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:40:33,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:40:33,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:40:35,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:40,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:40,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:40:40,618 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:40:40,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:40:40,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:40:40,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:40:42,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:40:45,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:47,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:40:48,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:40:48,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:40:48,975 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:40:50,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:55,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:40:56,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:40:56,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:40:56,786 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:40:56,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:40:56,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:40:58,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:41:01,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:02,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:41:03,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:41:03,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:41:07,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:12,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:12,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:41:12,737 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:41:12,737 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:41:12,738 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:41:13,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:41:15,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:41:17,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:17,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:41:18,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:41:18,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:41:18,979 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:41:22,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:27,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:28,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:41:28,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:41:28,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:41:28,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:41:29,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:41:31,016 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:41:32,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:41:33,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:41:33,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:41:33,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:38,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:43,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:44,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:41:44,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:41:44,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:41:44,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:41:45,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:41:47,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:41:47,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:41:48,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:41:48,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:41:48,980 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:41:48,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:53,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:41:58,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:01,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:42:01,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:42:01,081 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:42:01,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:42:02,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:42:02,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:42:03,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:42:03,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:42:03,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:42:04,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:09,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:14,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:17,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:42:17,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:42:17,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:42:17,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:42:17,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:42:18,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:42:18,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:42:18,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:42:18,983 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:42:19,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:42:19,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:24,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:29,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:32,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:42:33,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:42:33,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:42:33,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:42:33,424 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:42:33,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:42:33,426 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:42:34,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:42:35,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:42:35,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:40,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:45,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:47,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:42:48,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:42:48,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:42:48,986 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:42:49,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:42:49,133 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:42:49,134 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:42:49,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:42:50,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:42:51,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:42:51,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:42:56,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:01,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:02,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:43:03,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:43:03,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:43:05,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:43:05,105 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:43:05,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:43:05,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:43:06,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:43:07,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:43:07,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:12,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:17,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:17,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:43:18,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:43:18,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:43:18,990 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:43:21,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:43:21,268 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:43:21,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:43:21,270 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:43:22,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:43:22,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:23,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:43:27,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:32,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:32,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:43:33,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:43:33,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:43:37,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:43:37,333 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:43:37,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:43:37,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:37,335 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:43:38,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:43:39,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:43:42,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:47,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:47,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:43:48,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:43:48,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:43:48,992 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:43:52,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:43:53,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:43:53,304 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:43:53,304 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:43:53,306 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:43:54,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:43:55,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:43:58,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:02,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:44:03,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:44:03,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:44:03,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:08,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:09,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:44:09,411 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:44:09,412 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:44:09,414 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:44:10,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:44:11,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:44:14,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:17,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:44:18,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:44:18,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:44:18,995 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:44:19,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:24,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:25,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:44:25,446 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:44:25,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:44:25,447 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:44:26,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:44:27,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:44:30,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:32,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:44:33,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:44:33,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:44:36,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:41,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:41,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:44:41,668 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:44:41,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:44:41,670 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:44:42,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:44:43,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:44:46,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:47,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:44:48,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:44:48,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:44:49,000 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:44:52,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:57,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:44:57,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:44:57,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:44:57,649 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:44:57,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:44:58,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:44:59,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:45:02,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:02,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:45:03,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:45:03,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:45:08,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:13,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:13,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:45:13,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:45:13,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:45:13,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:45:14,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:45:15,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:45:17,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:45:18,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:45:18,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:45:19,001 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:45:19,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:24,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:29,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:30,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:45:30,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:45:30,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:45:30,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:45:30,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:45:31,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:45:32,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:45:33,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:45:33,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:45:34,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:39,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:44,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:46,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:45:46,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:45:46,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:45:46,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:45:46,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:45:47,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:45:47,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:45:48,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:45:48,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:45:49,004 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:45:50,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:45:55,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:00,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:02,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:46:02,115 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:46:02,115 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:46:02,117 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:46:02,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:46:02,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:46:03,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:46:03,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:46:03,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:46:05,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:10,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:15,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:17,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:46:18,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:46:18,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:46:18,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:46:18,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:46:18,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:46:18,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:46:19,007 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:46:19,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:46:19,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:46:21,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:26,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:31,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:32,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:46:33,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:46:33,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:46:34,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:46:34,533 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:46:34,534 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:46:34,535 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:46:35,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:46:36,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:37,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:46:41,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:46,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:47,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:46:48,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:46:48,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:46:49,011 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:46:50,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:46:50,597 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:46:50,598 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:46:50,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:46:51,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:46:51,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:46:53,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:46:56,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:01,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:02,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:47:03,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:47:03,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:47:06,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:47:06,854 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:47:06,854 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:47:06,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:06,856 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:47:07,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:47:09,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:47:11,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:16,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:17,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:47:18,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:47:18,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:47:19,013 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:47:22,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:22,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:47:22,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:47:22,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:47:22,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:47:23,316 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:47:25,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:47:27,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:32,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:47:33,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:47:33,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:47:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:38,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:39,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:47:39,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:47:39,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:47:39,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:47:39,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:47:41,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:47:44,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:47,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:47:48,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:47:48,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:47:49,016 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:47:50,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:55,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:47:55,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:47:55,364 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:47:55,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:47:55,366 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:47:56,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:47:57,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:48:00,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:02,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:48:03,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:48:03,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:48:05,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:10,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:11,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:48:11,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:48:11,706 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:48:11,708 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:48:12,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:48:13,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:48:15,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:17,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:48:18,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:48:18,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:48:19,020 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:48:21,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:26,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:27,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:48:27,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:48:27,667 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:48:27,668 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:48:28,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:48:29,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:48:31,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:32,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:48:33,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:48:33,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:48:37,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:42,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:44,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:48:44,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:48:44,086 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:48:44,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:48:44,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:48:45,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:48:47,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:47,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:48:48,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:48:48,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:48:49,022 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:48:53,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:48:58,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:00,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:49:00,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:49:00,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:49:00,423 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:49:01,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:49:01,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:49:02,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:49:03,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:49:03,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:49:03,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:08,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:13,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:16,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:49:16,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:49:16,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:49:16,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:49:16,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:49:17,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:49:17,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:49:18,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:49:18,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:49:18,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:19,025 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:49:24,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:29,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:32,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:49:32,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:49:32,570 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:49:32,570 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:49:32,572 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:49:33,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:49:33,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:49:33,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:49:33,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:49:34,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:39,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:44,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:47,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:49:48,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:49:48,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:49:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:49:48,716 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:49:48,717 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:49:48,718 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:49:49,029 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:49:49,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:49:50,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:49:51,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:49:55,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:00,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:02,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:50:03,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:50:03,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:50:04,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:50:04,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:50:04,755 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:50:04,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:50:05,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:50:05,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:07,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:50:10,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:15,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:17,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:50:18,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:50:18,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:50:19,032 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:50:20,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:50:20,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:50:20,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:50:20,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:20,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:50:21,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:50:23,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:50:25,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:30,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:32,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:50:33,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:50:33,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:50:36,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:36,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:50:36,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:50:36,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:50:36,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:50:37,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:50:39,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:50:41,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:46,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:47,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:50:48,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:50:48,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:50:49,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:50:52,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:50:52,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:50:52,852 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:50:52,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:50:52,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:50:53,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:50:55,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:50:57,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:02,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:51:03,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:51:03,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:51:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:08,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:08,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:51:08,996 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:51:08,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:51:08,998 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:51:09,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:51:11,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:51:13,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:17,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:51:18,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:51:18,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:51:19,037 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:51:19,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:24,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:24,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:51:24,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:51:24,850 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:51:24,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:51:25,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:51:27,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:51:29,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:32,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:51:33,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:51:33,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:51:35,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:40,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:40,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:51:40,933 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:51:40,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:51:40,935 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:51:41,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:51:43,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:51:45,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:47,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:51:48,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:51:48,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:51:49,042 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:51:51,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:56,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:51:56,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:51:56,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:51:56,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:51:56,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:51:57,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:51:59,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:52:01,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:02,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:52:03,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:52:03,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:52:07,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:12,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:13,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:52:13,092 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:52:13,093 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:52:13,094 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:52:13,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:52:15,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:52:17,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:17,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:52:18,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:52:18,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:52:19,043 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:52:23,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:28,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:29,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:52:29,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:52:29,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:52:29,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:52:29,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:52:31,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:52:32,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:52:33,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:52:33,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:52:33,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:38,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:43,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:45,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:52:45,356 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:52:45,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:52:45,358 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:52:45,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:52:47,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:52:47,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:52:48,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:52:48,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:52:49,046 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:52:49,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:54,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:52:59,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:01,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:53:01,493 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:53:01,494 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:53:01,496 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:53:01,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:53:02,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:53:03,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:53:03,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:53:03,614 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:53:04,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:09,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:14,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:17,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:53:17,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:53:17,549 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:53:17,550 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:53:17,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:53:17,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:53:18,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:53:18,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:53:19,051 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:53:19,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:53:20,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:25,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:30,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:32,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:53:33,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:53:33,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:53:34,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:53:34,030 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:53:34,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:53:34,032 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:53:34,640 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:53:35,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:53:36,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:41,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:46,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:47,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:53:48,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:53:48,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:53:49,052 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:53:50,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:53:50,185 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:53:50,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:53:50,187 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:53:50,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:53:51,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:53:51,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:53:56,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:01,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:02,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:54:03,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:54:03,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:54:06,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:54:06,253 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:54:06,253 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:54:06,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:06,255 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:54:06,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:54:07,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:54:11,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:16,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:17,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:54:18,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:54:18,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:54:19,055 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:54:22,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:22,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:54:22,213 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:54:22,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:54:22,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:54:22,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:54:23,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:54:27,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:32,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:32,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:54:33,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:54:33,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:54:37,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:38,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:54:38,150 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:54:38,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:54:38,152 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:54:38,695 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:54:39,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:54:43,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:47,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:54:48,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:54:48,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:54:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:49,058 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:54:54,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:54:54,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:54:54,371 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:54:54,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:54:54,373 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:54:54,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:54:55,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:54:59,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:02,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:55:03,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:55:03,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:55:05,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:10,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:55:10,237 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:55:10,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:55:10,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:55:10,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:55:11,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:11,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:55:16,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:17,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:55:18,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:55:18,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:55:19,061 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:55:22,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:26,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:55:26,284 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:55:26,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:55:26,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:55:26,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:55:27,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:27,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:55:32,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:32,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:55:33,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:55:33,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:55:37,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:42,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:42,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:55:42,446 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:55:42,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:55:42,448 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:55:42,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:55:43,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:55:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:47,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:55:48,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:55:48,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:55:49,065 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:55:53,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:58,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:55:58,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:55:58,634 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:55:58,635 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:55:58,636 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:55:58,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:55:59,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:56:02,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:56:03,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:56:03,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:56:03,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:08,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:13,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:14,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:56:14,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:56:14,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:56:14,786 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:56:15,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:56:15,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:56:17,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:56:18,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:56:18,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:56:18,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:19,068 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:56:24,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:29,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:30,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:56:30,951 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:56:30,952 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:56:30,953 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:56:31,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:56:31,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:56:32,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:56:33,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:56:33,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:56:34,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:39,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:44,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:47,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:56:47,008 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:56:47,008 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:56:47,009 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:56:47,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:56:47,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:56:47,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:56:48,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:56:48,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:56:49,072 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:56:50,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:56:55,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:00,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:02,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:57:03,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:57:03,140 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:57:03,141 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:57:03,142 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:57:03,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:57:03,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:57:03,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:57:03,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:57:05,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:10,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:15,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:17,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:57:18,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:57:18,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:57:19,074 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:57:19,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:57:19,316 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:57:19,317 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:57:19,318 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:57:19,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:57:21,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:21,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:57:26,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:31,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:32,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:57:33,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:57:33,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:57:35,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:57:35,680 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:57:35,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:57:35,682 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:57:35,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:57:36,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:37,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:57:41,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:46,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:47,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:57:48,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:57:48,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:57:49,077 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:57:51,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:57:51,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:57:51,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:57:51,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:57:51,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:57:52,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:57:53,863 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:57:56,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:01,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:02,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:58:03,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:58:03,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:58:07,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:07,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:58:07,966 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:58:07,966 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:58:07,968 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:58:08,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:58:09,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:58:12,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:17,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:58:18,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:58:18,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:58:18,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:19,080 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:58:24,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:24,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:58:24,487 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:58:24,487 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:58:24,489 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:58:24,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:58:25,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:58:29,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:32,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:58:33,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:58:33,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:58:35,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:40,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:58:40,352 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:58:40,352 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:58:40,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:58:40,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:58:41,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:41,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:58:46,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:47,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:58:48,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:58:48,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:58:49,082 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:58:52,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:56,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:58:56,854 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:58:56,855 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:58:56,856 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:58:56,920 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:58:57,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:58:57,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:59:02,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:59:03,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:59:03,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:59:03,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:08,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:13,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:59:13,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:59:13,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:59:13,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:59:13,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:59:13,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:59:14,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:17,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:59:18,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:59:18,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:59:19,085 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:59:20,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:25,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:29,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:59:29,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:59:29,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:59:29,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:59:29,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:59:29,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:59:30,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:32,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:59:33,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:59:33,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:59:35,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:40,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:45,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 01:59:45,316 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 01:59:45,317 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 01:59:45,318 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 01:59:45,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 01:59:45,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 01:59:46,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:47,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 01:59:48,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 01:59:48,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 01:59:49,088 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 01:59:52,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 01:59:57,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:01,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:00:01,360 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:00:01,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:00:01,362 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:00:01,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:00:01,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:00:02,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:02,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:00:03,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:00:03,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:00:08,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:13,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:17,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:00:17,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:00:17,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:00:17,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:00:17,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:00:17,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:00:17,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:00:18,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:00:18,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:00:18,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:19,092 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:00:24,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:29,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:32,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:00:33,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:00:33,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:00:33,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:00:33,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:00:33,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:00:33,427 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:00:34,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:00:34,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:36,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:00:39,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:44,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:47,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:00:48,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:00:48,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:00:49,095 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:00:49,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:00:49,233 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:00:49,234 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:00:49,235 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:00:50,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:00:50,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:00:50,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:00:55,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:00,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:02,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:01:03,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:01:03,163 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:01:05,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:01:05,293 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:01:05,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:01:05,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:05,295 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:01:06,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:06,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:01:10,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:15,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:17,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:01:18,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:01:18,164 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:01:19,098 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:01:21,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:21,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:01:21,456 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:01:21,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:01:21,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:01:22,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:22,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:01:26,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:26,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:30,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:31,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:32,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:32,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:32,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:01:33,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:01:33,250 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:01:38,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:38,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:40,069 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:42,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:01:43,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:44,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:01:47,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:01:48,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:01:48,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:01:48,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:49,102 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:01:54,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:01:58,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:01:58,689 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:01:58,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:01:58,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:01:59,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:01:59,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:00,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:02:02,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:02:03,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:02:03,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:02:05,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:10,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:14,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:02:14,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:02:14,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:02:14,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:02:15,095 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:02:15,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:16,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:02:17,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:02:18,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:02:18,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:02:19,104 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:02:21,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:26,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:31,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:02:31,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:02:31,050 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:02:31,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:02:31,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:02:32,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:32,107 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:02:32,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:02:33,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:02:33,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:02:37,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:42,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:47,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:02:47,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:02:47,167 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:02:47,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:02:47,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:02:47,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:48,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:02:48,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:02:48,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:02:48,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:02:49,107 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:02:53,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:02:58,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:02,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:03:03,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:03:03,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:03:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:03:03,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:03:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:03,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:03:03,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:03:04,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:03:04,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:03:08,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:13,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:17,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:03:18,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:03:18,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:03:18,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:19,110 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:03:19,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:03:19,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:03:19,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:03:19,587 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:03:20,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:03:20,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:03:23,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:28,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:32,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:03:33,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:03:33,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:03:34,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:35,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:03:35,528 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:03:35,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:03:35,530 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:03:36,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:03:36,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:03:39,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:44,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:47,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:03:48,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:03:48,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:03:49,113 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:03:50,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:03:51,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:03:51,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:03:51,669 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:03:51,671 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:03:52,171 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:03:54,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:03:55,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:00,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:02,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:04:03,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:04:03,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:04:06,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:07,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:04:07,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:04:07,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:04:07,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:04:08,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:04:08,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:04:11,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:16,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:17,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:04:18,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:04:18,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:04:19,115 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:04:22,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:23,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:04:23,479 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:04:23,479 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:04:23,480 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:04:24,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:04:24,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:04:27,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:32,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:32,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:04:33,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:04:33,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:04:38,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:39,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:04:39,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:04:39,466 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:04:39,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:04:40,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:04:40,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:04:43,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:47,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:04:48,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:04:48,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:04:49,119 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:04:49,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:54,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:04:55,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:04:55,406 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:04:55,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:04:55,408 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:04:56,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:04:56,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:04:59,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:02,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:05:03,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:05:03,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:05:05,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:10,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:11,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:05:11,487 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:05:11,487 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:05:11,489 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:05:12,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:05:12,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:05:15,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:17,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:05:18,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:05:18,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:05:19,124 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:05:21,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:26,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:27,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:05:27,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:05:27,580 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:05:27,581 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:05:28,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:05:28,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:05:31,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:32,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:05:33,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:05:33,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:05:37,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:42,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:43,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:05:43,619 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:05:43,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:05:43,622 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:05:44,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:05:44,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:05:47,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:47,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:05:48,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:05:48,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:05:49,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:05:53,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:58,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:05:59,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:05:59,689 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:05:59,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:05:59,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:06:00,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:06:00,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:06:02,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:06:03,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:06:03,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:06:03,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:08,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:13,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:15,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:06:15,743 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:06:15,743 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:06:15,745 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:06:16,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:06:16,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:06:17,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:06:18,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:06:18,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:06:18,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:19,128 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:06:24,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:29,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:31,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:06:31,717 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:06:31,718 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:06:31,719 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:06:32,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:06:32,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:06:32,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:06:33,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:06:33,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:06:34,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:39,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:44,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:47,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:06:47,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:06:47,665 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:06:47,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:06:47,667 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:06:48,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:06:48,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:06:48,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:06:48,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:06:49,132 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:06:50,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:06:55,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:00,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:02,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:07:03,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:07:03,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:07:03,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:07:03,611 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:07:03,612 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:07:03,613 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:07:04,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:07:04,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:07:05,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:10,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:15,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:17,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:07:18,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:07:18,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:07:19,134 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:07:19,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:07:19,676 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:07:19,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:07:19,677 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:07:20,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:07:20,351 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:07:20,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:25,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:30,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:32,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:07:33,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:07:33,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:07:35,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:07:35,847 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:07:35,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:07:35,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:35,849 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:07:36,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:07:36,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:07:40,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:45,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:47,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:07:48,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:07:48,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:07:49,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:07:51,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:07:51,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:07:51,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:07:51,800 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:07:51,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:07:52,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:07:52,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:07:56,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:01,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:02,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:08:03,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:08:03,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:08:07,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:07,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:08:07,851 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:08:07,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:08:07,853 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:08:08,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:08:08,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:08:12,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:17,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:08:18,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:08:18,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:08:18,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:19,140 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:08:23,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:08:23,911 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:08:23,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:08:23,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:23,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:08:24,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:08:24,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:08:28,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:32,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:08:33,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:08:33,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:08:34,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:39,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:40,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:08:40,170 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:08:40,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:08:40,172 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:08:40,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:08:42,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:08:45,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:47,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:08:48,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:08:48,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:08:49,144 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:08:51,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:56,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:08:56,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:08:56,216 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:08:56,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:08:56,218 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:08:56,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:08:58,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:09:01,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:02,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:09:03,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:09:03,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:09:06,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:11,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:12,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:09:12,255 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:09:12,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:09:12,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:09:12,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:09:14,449 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:09:17,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:17,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:09:18,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:09:18,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:09:19,146 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:09:23,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:28,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:28,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:09:28,212 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:09:28,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:09:28,213 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:09:28,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:09:30,462 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:09:32,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:09:33,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:09:33,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:09:33,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:38,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:43,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:44,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:09:44,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:09:44,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:09:44,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:09:45,475 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:09:46,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:09:47,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:09:48,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:09:48,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:09:48,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:49,148 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:09:54,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:09:59,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:00,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:10:00,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:10:00,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:10:00,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:10:01,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:10:02,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:10:02,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:10:03,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:10:03,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:10:04,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:09,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:14,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:16,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:10:16,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:10:16,798 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:10:16,799 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:10:17,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:10:17,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:10:18,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:10:18,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:10:18,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:10:19,153 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:10:20,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:25,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:30,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:32,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:10:32,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:10:32,932 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:10:32,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:10:32,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:10:33,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:10:33,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:10:33,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:10:34,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:10:35,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:40,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:45,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:47,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:10:48,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:10:48,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:10:49,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:10:49,088 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:10:49,088 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:10:49,090 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:10:49,154 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:10:49,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:10:50,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:10:51,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:10:56,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:01,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:02,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:11:03,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:11:03,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:11:05,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:11:05,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:11:05,550 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:11:05,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:11:06,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:11:06,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:11:06,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:11,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:16,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:17,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:11:18,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:11:18,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:11:19,158 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:11:21,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:11:21,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:11:21,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:11:21,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:11:21,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:11:22,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:22,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:11:27,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:32,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:32,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:11:33,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:11:33,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:11:37,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:11:37,908 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:11:37,909 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:11:37,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:37,911 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:11:38,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:11:38,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:11:42,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:47,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:11:48,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:11:48,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:11:48,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:49,163 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:11:53,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:11:53,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:11:53,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:11:53,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:11:53,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:11:54,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:11:54,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:11:58,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:02,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:12:03,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:12:03,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:12:04,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:09,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:10,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:12:10,377 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:12:10,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:12:10,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:12:10,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:12:12,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:12:14,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:17,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:12:18,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:12:18,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:12:19,166 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:12:20,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:25,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:26,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:12:26,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:12:26,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:12:26,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:12:26,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:12:28,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:12:30,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:32,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:12:33,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:12:33,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:12:36,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:41,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:42,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:12:42,786 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:12:42,787 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:12:42,788 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:12:43,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:12:44,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:12:46,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:47,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:12:48,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:12:48,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:12:49,169 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:12:52,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:57,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:12:59,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:12:59,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:12:59,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:12:59,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:12:59,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:13:00,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:13:02,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:13:02,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:03,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:13:03,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:13:08,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:13,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:15,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:13:15,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:13:15,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:13:15,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:13:15,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:13:16,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:13:17,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:13:18,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:13:18,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:13:19,172 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:13:19,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:24,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:29,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:31,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:13:31,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:13:31,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:13:31,581 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:13:31,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:13:32,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:13:32,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:13:33,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:13:33,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:13:34,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:39,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:44,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:47,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:13:47,635 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:13:47,636 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:13:47,637 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:13:47,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:13:47,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:13:48,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:13:48,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:13:48,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:13:49,176 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:13:50,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:13:55,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:00,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:02,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:14:03,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:14:03,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:14:03,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:14:03,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:14:03,745 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:14:03,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:14:04,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:14:04,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:14:05,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:10,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:15,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:17,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:14:18,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:14:18,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:14:19,178 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:14:19,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:14:19,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:14:19,593 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:14:19,594 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:14:19,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:14:20,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:14:21,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:26,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:31,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:32,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:14:33,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:14:33,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:14:35,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:14:35,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:14:35,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:14:35,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:14:35,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:14:36,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:14:37,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:42,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:47,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:47,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:14:48,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:14:48,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:14:49,181 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:14:51,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:14:51,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:14:51,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:14:51,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:14:51,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:14:52,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:14:53,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:14:58,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:02,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:15:03,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:15:03,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:15:04,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:07,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:15:07,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:15:07,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:15:07,396 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:15:07,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:15:08,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:15:09,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:14,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:17,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:15:18,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:15:18,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:15:19,183 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:15:20,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:23,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:15:23,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:15:23,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:15:23,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:15:23,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:15:24,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:15:25,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:30,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:32,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:15:33,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:15:33,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:15:36,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:39,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:15:39,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:15:39,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:15:39,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:15:39,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:15:40,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:15:41,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:46,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:47,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:15:48,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:15:48,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:15:49,187 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:15:52,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:15:55,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:15:55,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:15:55,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:15:55,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:15:56,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:15:56,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:15:57,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:02,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:16:03,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:16:03,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:16:03,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:08,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:11,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:16:11,867 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:16:11,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:16:11,869 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:16:12,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:16:12,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:16:13,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:17,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:16:18,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:16:18,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:16:19,190 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:16:19,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:24,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:27,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:16:27,915 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:16:27,916 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:16:27,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:16:28,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:16:28,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:16:29,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:32,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:16:33,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:16:33,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:16:35,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:40,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:43,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:16:43,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:16:43,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:16:43,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:16:43,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:16:44,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:16:45,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:47,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:16:48,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:16:48,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:16:49,193 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:16:51,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:56,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:16:59,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:16:59,734 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:16:59,735 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:16:59,736 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:16:59,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:17:00,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:17:01,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:02,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:17:03,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:17:03,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:17:07,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:12,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:15,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:17:15,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:17:15,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:17:15,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:17:15,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:17:16,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:17:17,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:17,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:17:18,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:17:18,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:17:19,197 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:17:23,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:28,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:31,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:17:31,858 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:17:31,858 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:17:31,860 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:17:31,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:17:32,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:17:32,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:17:33,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:17:33,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:17:33,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:38,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:43,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:47,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:17:47,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:17:47,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:17:47,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:17:47,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:17:48,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:17:48,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:17:48,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:17:48,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:17:49,211 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:17:49,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:54,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:17:59,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:02,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:18:03,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:18:03,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:18:03,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:18:03,866 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:18:03,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:18:03,868 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:18:03,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:18:04,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:04,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:18:09,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:14,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:17,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:18:18,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:18:18,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:18:19,214 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:18:19,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:18:19,928 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:18:19,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:18:19,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:19,930 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:18:20,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:18:20,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:18:24,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:29,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:32,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:18:33,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:18:33,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:18:35,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:35,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:18:35,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:18:35,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:18:35,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:18:36,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:18:36,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:18:40,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:45,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:47,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:18:48,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:18:48,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:18:49,219 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:18:51,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:18:52,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:18:52,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:18:52,235 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:18:52,237 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:18:52,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:18:52,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:18:56,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:01,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:02,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:19:03,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:19:03,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:19:06,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:08,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:19:08,268 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:19:08,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:19:08,270 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:19:08,947 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:19:08,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:19:12,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:17,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:17,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:19:18,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:19:18,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:19:19,221 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:19:23,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:24,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:19:24,307 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:19:24,307 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:19:24,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:19:24,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:19:24,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:19:28,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:32,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:19:33,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:19:33,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:19:33,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:38,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:40,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:19:40,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:19:40,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:19:40,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:19:40,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:19:40,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:19:44,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:47,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:19:48,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:19:48,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:19:49,223 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:19:49,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:54,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:19:56,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:19:56,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:19:56,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:19:56,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:19:56,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:19:56,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:19:59,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:02,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:20:03,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:20:03,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:20:05,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:10,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:12,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:20:12,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:20:12,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:20:12,682 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:20:13,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:20:13,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:20:15,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:17,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:20:18,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:20:18,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:20:19,226 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:20:21,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:26,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:28,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:20:28,722 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:20:28,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:20:28,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:20:29,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:20:29,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:20:31,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:32,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:20:33,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:20:33,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:20:37,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:42,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:45,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:20:45,014 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:20:45,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:20:45,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:20:45,028 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:20:47,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:20:47,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:20:47,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:48,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:20:48,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:20:49,743 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:20:53,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:20:58,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:01,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:21:01,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:21:01,081 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:21:01,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:21:02,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:21:02,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:21:03,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:21:03,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:21:03,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:21:04,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:09,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:14,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:17,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:21:17,448 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:21:17,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:21:17,449 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:21:17,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:21:18,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:21:18,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:21:18,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:21:19,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:21:19,234 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:21:20,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:25,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:30,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:32,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:21:33,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:21:33,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:21:33,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:21:33,493 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:21:33,493 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:21:33,496 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:21:34,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:21:35,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:21:35,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:40,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:45,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:47,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:21:48,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:21:48,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:21:49,237 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:21:49,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:21:49,878 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:21:49,879 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:21:49,880 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:21:50,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:21:50,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:21:51,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:21:55,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:00,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:02,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:22:03,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:22:03,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:22:06,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:22:06,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:22:06,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:22:06,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:06,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:22:06,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:22:07,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:22:11,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:16,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:17,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:22:18,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:22:18,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:22:19,241 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:22:21,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:22,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:22:22,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:22:22,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:22:22,308 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:22:23,111 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:22:23,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:22:26,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:31,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:32,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:22:33,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:22:33,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:22:36,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:38,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:22:38,449 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:22:38,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:22:38,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:22:39,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:22:39,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:22:41,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:46,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:47,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:22:48,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:22:48,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:22:49,243 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:22:52,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:22:54,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:22:54,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:22:54,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:22:54,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:22:55,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:22:55,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:22:57,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:02,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:02,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:23:03,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:23:03,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:23:08,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:10,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:23:10,890 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:23:10,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:23:10,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:23:11,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:23:11,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:23:13,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:17,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:23:18,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:23:18,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:23:19,246 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:23:19,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:24,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:27,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:23:27,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:23:27,113 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:23:27,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:23:27,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:23:29,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:23:30,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:32,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:23:33,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:23:33,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:23:35,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:40,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:43,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:23:43,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:23:43,426 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:23:43,429 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:23:44,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:23:45,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:23:45,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:47,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:23:48,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:23:48,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:23:49,249 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:23:51,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:56,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:23:59,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:23:59,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:23:59,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:23:59,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:24:00,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:24:01,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:24:01,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:02,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:24:03,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:24:03,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:24:07,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:12,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:15,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:24:15,696 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:24:15,697 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:24:15,699 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:24:16,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:24:17,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:24:17,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:17,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:24:18,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:24:18,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:24:19,253 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:24:23,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:28,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:31,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:24:31,635 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:24:31,635 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:24:31,637 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:24:32,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:24:32,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:24:33,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:24:33,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:24:33,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:24:33,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:38,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:43,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:47,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:24:47,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:24:47,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:24:47,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:24:47,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:24:48,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:24:48,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:24:48,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:24:49,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:24:49,255 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:24:49,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:54,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:24:59,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:02,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:25:03,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:25:03,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:25:03,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:25:03,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:25:03,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:25:03,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:25:04,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:25:04,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:05,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:25:09,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:14,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:17,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:25:18,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:25:18,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:25:19,259 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:25:19,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:25:19,419 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:25:19,420 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:25:19,421 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:25:20,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:25:20,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:21,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:25:25,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:30,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:32,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:25:33,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:25:33,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:25:35,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:35,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:25:35,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:25:35,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:25:35,581 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:25:36,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:25:37,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:25:40,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:45,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:47,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:25:48,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:25:48,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:25:49,263 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:25:51,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:25:51,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:25:51,523 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:25:51,524 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:25:51,525 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:25:52,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:25:53,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:25:56,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:01,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:02,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:26:03,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:26:03,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:26:07,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:07,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:26:07,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:26:07,558 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:26:07,560 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:26:08,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:26:09,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:26:12,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:17,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:17,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:26:18,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:26:18,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:26:19,265 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:26:23,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:23,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:26:23,607 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:26:23,608 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:26:23,609 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:26:24,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:26:25,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:26:28,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:32,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:26:33,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:26:33,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:26:34,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:39,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:39,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:26:39,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:26:39,755 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:26:39,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:26:40,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:26:41,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:26:44,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:47,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:26:48,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:26:48,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:26:49,267 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:26:50,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:55,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:26:55,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:26:55,815 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:26:55,816 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:26:55,817 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:26:56,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:26:57,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:27:00,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:02,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:27:03,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:27:03,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:27:06,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:11,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:11,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:27:11,678 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:27:11,679 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:27:11,680 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:27:12,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:27:13,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:27:16,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:17,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:27:18,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:27:18,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:27:19,270 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:27:22,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:27,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:27,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:27:27,515 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:27:27,516 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:27:27,517 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:27:28,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:27:29,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:27:32,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:32,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:27:33,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:27:33,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:27:38,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:43,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:43,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:27:43,745 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:27:43,745 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:27:43,747 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:27:44,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:27:45,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:27:47,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:27:48,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:27:48,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:27:48,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:49,273 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:27:54,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:59,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:27:59,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:27:59,803 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:27:59,803 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:27:59,805 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:28:00,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:28:01,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:28:02,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:28:03,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:28:03,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:28:04,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:09,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:14,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:15,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:28:15,659 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:28:15,659 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:28:15,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:28:16,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:28:17,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:28:17,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:28:18,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:28:18,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:28:19,276 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:28:20,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:25,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:30,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:31,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:28:31,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:28:31,811 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:28:31,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:28:32,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:28:32,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:28:33,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:28:33,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:28:33,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:28:35,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:40,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:45,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:47,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:28:48,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:28:48,005 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:28:48,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:28:48,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:28:48,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:28:48,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:28:48,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:28:49,279 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:28:49,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:28:51,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:28:56,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:01,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:02,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:29:03,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:29:03,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:29:04,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:29:04,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:29:04,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:29:04,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:29:04,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:29:05,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:29:07,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:12,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:17,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:17,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:29:18,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:29:18,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:29:19,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:29:20,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:29:20,182 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:29:20,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:29:20,185 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:29:20,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:29:21,470 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:29:23,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:28,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:32,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:29:33,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:29:33,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:29:33,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:36,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:29:36,313 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:29:36,314 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:29:36,315 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:29:36,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:29:37,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:29:39,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:44,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:47,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:29:48,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:29:48,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:29:49,285 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:29:50,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:29:52,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:29:52,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:29:52,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:29:52,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:29:52,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:29:53,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:29:55,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:00,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:02,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:30:03,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:30:03,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:30:06,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:08,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:30:08,602 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:30:08,602 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:30:08,604 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:30:09,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:30:09,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:30:11,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:16,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:17,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:30:18,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:30:18,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:30:19,289 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:30:22,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:24,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:30:24,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:30:24,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:30:24,770 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:30:25,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:30:26,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:30:27,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:32,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:32,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:30:33,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:30:33,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:30:38,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:40,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:30:40,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:30:40,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:30:40,828 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:30:41,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:30:42,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:30:43,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:47,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:30:48,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:30:48,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:30:49,293 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:30:49,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:54,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:30:57,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:30:57,079 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:30:57,079 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:30:57,081 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:30:57,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:30:58,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:31:00,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:02,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:31:03,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:31:03,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:31:05,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:10,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:13,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:31:13,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:31:13,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:31:13,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:31:13,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:31:14,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:31:16,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:17,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:31:18,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:31:18,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:31:19,295 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:31:22,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:27,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:29,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:31:29,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:31:29,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:31:29,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:31:30,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:31:32,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:31:32,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:32,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:31:33,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:31:33,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:31:38,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:43,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:45,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:31:45,827 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:31:45,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:31:45,829 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:31:46,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:31:47,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:31:48,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:31:48,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:31:48,602 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:31:49,298 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:31:49,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:54,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:31:59,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:02,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:32:02,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:32:02,132 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:32:02,134 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:32:02,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:32:02,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:32:03,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:32:03,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:32:04,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:04,615 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:32:09,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:14,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:17,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:32:18,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:32:18,060 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:32:18,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:32:18,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:32:18,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:32:18,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:32:18,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:32:19,301 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:32:20,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:20,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:32:25,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:30,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:32,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:32:33,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:32:33,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:32:34,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:32:34,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:32:34,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:32:34,321 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:32:34,640 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:32:35,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:36,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:32:40,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:45,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:47,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:32:48,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:32:48,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:32:49,305 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:32:50,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:32:50,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:32:50,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:32:50,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:32:50,653 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:32:51,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:32:52,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:32:56,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:01,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:02,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:33:03,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:33:03,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:33:06,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:06,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:33:06,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:33:06,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:33:06,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:33:06,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:33:08,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:33:11,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:16,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:17,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:33:18,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:33:18,193 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:33:19,307 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:33:22,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:22,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:33:22,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:33:22,477 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:33:22,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:33:22,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:33:24,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:33:27,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:32,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:32,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:33:33,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:33:33,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:33:38,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:38,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:33:38,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:33:38,587 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:33:38,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:33:38,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:33:40,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:33:43,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:47,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:33:48,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:33:48,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:33:49,310 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:33:49,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:54,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:33:54,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:33:54,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:33:54,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:33:54,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:33:55,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:33:56,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:33:59,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:02,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:34:03,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:34:03,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:34:05,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:10,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:10,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:34:10,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:34:10,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:34:10,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:34:11,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:34:12,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:34:15,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:17,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:34:18,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:34:18,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:34:19,315 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:34:21,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:26,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:27,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:34:27,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:34:27,301 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:34:27,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:34:27,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:34:28,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:34:32,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:32,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:34:33,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:34:33,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:34:37,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:42,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:43,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:34:43,298 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:34:43,298 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:34:43,300 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:34:43,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:34:44,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:34:47,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:34:47,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:48,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:34:48,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:34:49,316 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:34:53,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:58,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:34:59,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:34:59,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:34:59,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:34:59,256 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:34:59,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:35:00,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:35:02,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:35:03,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:35:03,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:35:03,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:08,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:13,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:15,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:35:15,297 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:35:15,298 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:35:15,299 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:35:15,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:35:16,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:35:17,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:35:18,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:35:18,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:35:18,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:19,319 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:35:24,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:29,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:31,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:35:31,258 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:35:31,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:35:31,260 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:35:31,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:35:32,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:35:32,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:35:33,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:35:33,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:35:34,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:39,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:44,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:47,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:35:47,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:35:47,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:35:47,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:35:47,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:35:47,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:35:48,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:35:48,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:35:48,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:35:49,324 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:35:50,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:35:55,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:00,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:02,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:36:03,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:36:03,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:36:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:36:03,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:36:03,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:36:03,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:36:03,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:36:04,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:36:05,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:10,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:15,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:17,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:36:18,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:36:18,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:36:19,325 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:36:19,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:36:19,521 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:36:19,521 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:36:19,523 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:36:19,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:36:20,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:20,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:36:25,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:30,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:32,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:36:33,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:36:33,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:36:35,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:36:35,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:36:35,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:36:35,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:35,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:36:35,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:36:36,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:36:40,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:45,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:47,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:36:48,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:36:48,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:36:49,328 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:36:51,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:36:51,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:36:51,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:36:51,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:36:51,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:36:51,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:36:52,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:36:56,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:01,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:02,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:37:03,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:37:03,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:37:07,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:07,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:37:07,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:37:07,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:37:07,677 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:37:07,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:37:08,874 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:37:12,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:17,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:17,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:37:18,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:37:18,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:37:19,331 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:37:23,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:23,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:37:23,531 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:37:23,531 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:37:23,533 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:37:23,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:37:24,887 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:37:28,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:32,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:37:33,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:37:33,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:37:34,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:39,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:39,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:37:39,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:37:39,487 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:37:39,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:37:39,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:37:40,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:37:44,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:47,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:37:48,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:37:48,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:37:49,336 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:37:50,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:55,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:37:55,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:37:55,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:37:55,625 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:37:55,626 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:37:55,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:37:56,914 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:38:00,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:02,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:38:03,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:38:03,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:38:06,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:11,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:11,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:38:11,936 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:38:11,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:38:11,938 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:38:12,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:38:14,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:38:16,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:17,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:38:18,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:38:18,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:38:19,336 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:38:22,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:27,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:27,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:38:27,792 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:38:27,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:38:27,794 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:38:27,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:38:28,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:38:32,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:32,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:38:33,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:38:33,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:38:38,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:43,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:43,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:38:43,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:38:43,945 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:38:43,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:38:43,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:38:44,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:38:47,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:38:48,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:38:48,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:38:48,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:49,340 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:38:54,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:59,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:38:59,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:38:59,993 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:38:59,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:38:59,995 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:39:00,971 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:39:02,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:39:02,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:39:03,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:39:03,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:39:04,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:09,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:14,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:16,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:39:16,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:39:16,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:39:16,249 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:39:16,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:39:17,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:39:18,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:39:18,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:39:18,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:39:19,375 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:39:20,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:25,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:30,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:32,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:39:32,202 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:39:32,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:39:32,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:39:32,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:39:33,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:39:33,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:39:33,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:39:35,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:39:35,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:40,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:45,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:47,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:39:48,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:39:48,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:39:48,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:39:48,441 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:39:48,441 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:39:48,443 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:39:49,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:39:49,376 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:39:51,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:39:51,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:39:56,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:01,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:02,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:40:03,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:40:03,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:40:04,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:40:04,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:40:04,439 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:40:04,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:40:05,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:40:06,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:07,028 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:40:11,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:16,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:17,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:40:18,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:40:18,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:40:19,379 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:40:20,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:40:20,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:40:20,550 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:40:20,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:40:21,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:40:21,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:23,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:40:26,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:31,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:32,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:40:33,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:40:33,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:40:36,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:40:36,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:40:36,800 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:40:36,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:36,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:40:37,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:40:39,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:40:41,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:46,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:47,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:40:48,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:40:48,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:40:49,382 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:40:52,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:40:52,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:40:52,742 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:40:52,742 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:40:52,744 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:40:53,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:40:55,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:40:57,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:02,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:02,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:41:03,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:41:03,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:41:08,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:08,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:41:08,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:41:08,997 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:41:08,999 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:41:09,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:41:11,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:41:14,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:17,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:41:18,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:41:18,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:41:19,385 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:41:19,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:24,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:25,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:41:25,126 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:41:25,127 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:41:25,128 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:41:26,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:41:27,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:41:30,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:32,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:41:33,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:41:33,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:41:35,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:40,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:41,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:41:41,360 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:41:41,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:41:41,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:41:42,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:41:43,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:41:46,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:47,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:41:48,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:41:48,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:41:49,388 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:41:51,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:56,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:41:57,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:41:57,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:41:57,442 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:41:57,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:41:58,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:41:59,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:42:01,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:02,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:42:03,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:42:03,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:42:07,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:12,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:13,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:42:13,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:42:13,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:42:13,828 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:42:14,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:42:15,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:42:17,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:42:17,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:18,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:42:18,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:42:19,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:42:23,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:28,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:29,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:42:29,780 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:42:29,781 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:42:29,783 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:42:30,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:42:31,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:42:32,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:42:33,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:42:33,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:42:34,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:39,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:44,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:46,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:42:46,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:42:46,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:42:46,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:42:46,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:42:47,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:42:47,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:42:48,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:42:48,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:42:49,396 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:42:49,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:54,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:42:59,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:02,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:43:02,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:43:02,090 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:43:02,092 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:43:02,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:43:02,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:43:03,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:43:03,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:43:03,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:43:04,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:09,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:14,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:17,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:43:18,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:43:18,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:43:18,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:43:18,406 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:43:18,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:43:18,408 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:43:19,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:43:19,398 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:43:20,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:21,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:43:25,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:30,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:32,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:43:33,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:43:33,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:43:34,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:43:34,183 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:43:34,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:43:34,185 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:43:34,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:43:35,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:43:36,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:41,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:46,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:47,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:43:48,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:43:48,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:43:49,400 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:43:50,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:43:50,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:43:50,522 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:43:50,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:43:51,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:43:51,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:43:53,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:43:56,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:01,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:02,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:44:03,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:44:03,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:44:06,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:44:06,644 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:44:06,644 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:44:06,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:06,646 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:44:07,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:44:09,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:44:11,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:16,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:17,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:44:18,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:44:18,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:44:19,403 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:44:22,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:22,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:44:22,874 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:44:22,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:44:22,876 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:44:23,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:44:25,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:44:27,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:32,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:44:33,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:44:33,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:44:33,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:38,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:38,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:44:39,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:44:39,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:44:39,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:44:39,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:44:41,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:44:44,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:47,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:44:48,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:44:48,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:44:49,406 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:44:49,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:54,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:44:55,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:44:55,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:44:55,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:44:55,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:44:55,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:44:57,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:45:00,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:02,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:45:03,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:45:03,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:45:05,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:10,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:11,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:45:11,020 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:45:11,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:45:11,022 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:45:11,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:45:13,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:45:16,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:17,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:45:18,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:45:18,198 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:45:19,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:45:21,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:26,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:27,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:45:27,065 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:45:27,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:45:27,067 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:45:27,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:45:29,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:45:32,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:32,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:45:33,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:45:33,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:45:37,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:42,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:43,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:45:43,224 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:45:43,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:45:43,225 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:45:43,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:45:45,326 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:45:47,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:45:47,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:48,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:45:48,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:45:49,412 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:45:53,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:58,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:45:59,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:45:59,195 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:45:59,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:45:59,197 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:45:59,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:46:01,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:46:02,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:46:03,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:46:03,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:46:03,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:08,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:13,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:15,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:46:15,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:46:15,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:46:15,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:46:16,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:46:17,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:46:17,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:46:18,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:46:18,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:46:19,416 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:46:19,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:24,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:29,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:31,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:46:31,591 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:46:31,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:46:31,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:46:32,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:46:32,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:46:33,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:46:33,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:46:33,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:46:35,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:40,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:45,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:47,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:46:47,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:46:47,832 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:46:47,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:46:47,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:46:48,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:46:48,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:46:48,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:46:49,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:46:49,418 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:46:50,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:46:55,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:00,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:02,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:47:03,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:47:03,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:47:03,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:47:03,777 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:47:03,777 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:47:03,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:47:04,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:47:05,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:47:05,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:10,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:15,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:17,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:47:18,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:47:18,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:47:19,421 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:47:19,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:47:19,920 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:47:19,920 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:47:19,922 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:47:20,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:47:20,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:21,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:47:25,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:30,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:32,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:47:33,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:47:33,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:47:35,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:47:35,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:47:35,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:47:35,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:47:36,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:47:36,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:37,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:47:41,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:46,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:47,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:47:48,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:47:48,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:47:49,425 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:47:51,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:47:51,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:47:51,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:47:51,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:47:51,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:47:52,445 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:47:53,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:47:56,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:01,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:02,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:48:03,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:48:03,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:48:07,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:07,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:48:07,879 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:48:07,880 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:48:07,882 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:48:08,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:48:09,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:48:12,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:17,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:48:18,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:48:18,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:48:18,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:19,427 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:48:24,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:48:24,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:48:24,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:48:24,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:24,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:48:24,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:48:25,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:48:29,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:32,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:48:33,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:48:33,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:48:34,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:39,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:40,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:48:40,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:48:40,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:48:40,253 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:48:40,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:48:41,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:48:45,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:47,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:48:48,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:48:48,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:48:49,430 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:48:50,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:55,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:48:56,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:48:56,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:48:56,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:48:56,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:48:56,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:48:57,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:49:01,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:02,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:49:03,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:49:03,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:49:06,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:11,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:12,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:49:12,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:49:12,549 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:49:12,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:49:13,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:49:13,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:49:16,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:17,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:49:18,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:49:18,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:49:19,432 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:49:22,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:27,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:28,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:49:28,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:49:28,675 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:49:28,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:49:29,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:49:31,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:49:32,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:32,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:49:33,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:49:33,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:49:38,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:43,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:44,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:49:44,621 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:49:44,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:49:44,623 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:49:45,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:49:45,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:49:47,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:49:48,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:49:48,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:49:48,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:49,437 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:49:53,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:49:58,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:00,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:50:00,756 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:50:00,757 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:50:00,758 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:50:01,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:50:02,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:50:03,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:50:03,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:50:03,565 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:50:04,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:09,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:14,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:16,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:50:16,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:50:16,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:50:16,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:50:17,576 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:50:17,576 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:50:17,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:50:18,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:50:18,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:50:19,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:19,439 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:50:24,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:29,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:32,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:50:32,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:50:32,902 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:50:32,902 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:50:32,904 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:50:33,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:50:33,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:50:33,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:50:35,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:35,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:50:40,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:45,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:47,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:50:48,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:50:48,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:50:49,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:50:49,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:50:49,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:50:49,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:50:49,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:50:49,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:50:50,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:50:51,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:50:55,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:00,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:02,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:51:03,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:51:03,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:51:05,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:51:05,137 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:51:05,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:51:05,138 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:51:05,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:51:06,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:07,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:51:11,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:16,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:17,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:51:18,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:51:18,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:51:19,447 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:51:21,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:51:21,282 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:51:21,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:51:21,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:21,284 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:51:21,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:51:23,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:51:26,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:31,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:32,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:51:33,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:51:33,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:51:36,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:37,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:51:37,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:51:37,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:51:37,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:51:37,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:51:39,640 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:51:42,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:47,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:47,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:51:48,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:51:48,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:51:49,449 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:51:52,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:51:53,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:51:53,470 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:51:53,470 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:51:53,472 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:51:53,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:51:55,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:51:57,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:02,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:02,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:52:03,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:52:03,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:52:08,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:09,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:52:09,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:52:09,438 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:52:09,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:52:09,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:52:11,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:52:13,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:17,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:52:18,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:52:18,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:52:19,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:19,451 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:52:24,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:25,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:52:25,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:52:25,544 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:52:25,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:52:25,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:52:27,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:52:29,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:32,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:52:33,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:52:33,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:52:35,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:40,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:41,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:52:41,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:52:41,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:52:41,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:52:41,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:52:43,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:52:45,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:47,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:52:48,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:52:48,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:52:49,455 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:52:51,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:56,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:52:57,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:52:57,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:52:57,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:52:57,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:52:58,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:52:59,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:53:01,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:02,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:53:03,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:53:03,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:53:07,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:12,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:13,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:53:13,889 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:53:13,889 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:53:13,890 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:53:14,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:53:15,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:53:17,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:53:17,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:18,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:53:18,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:53:19,458 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:53:23,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:28,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:30,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:53:30,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:53:30,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:53:30,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:53:30,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:53:31,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:53:32,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:53:33,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:53:33,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:53:34,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:39,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:44,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:45,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:53:45,886 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:53:45,886 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:53:45,888 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:53:46,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:53:47,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:53:47,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:53:48,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:53:48,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:53:49,461 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:53:49,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:54,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:53:59,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:02,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:54:02,150 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:54:02,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:54:02,152 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:54:02,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:54:02,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:54:03,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:54:03,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:54:03,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:54:04,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:09,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:14,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:17,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:54:18,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:54:18,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:54:18,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:54:18,390 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:54:18,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:54:18,392 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:54:18,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:54:19,464 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:54:19,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:54:20,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:25,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:30,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:32,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:54:33,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:54:33,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:54:34,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:54:34,277 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:54:34,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:54:34,279 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:54:34,792 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:54:35,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:54:36,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:41,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:46,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:47,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:54:48,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:54:48,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:54:49,467 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:54:50,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:54:50,531 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:54:50,532 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:54:50,533 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:54:50,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:54:51,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:54:51,807 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:54:56,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:01,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:02,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:55:03,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:55:03,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:55:06,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:55:06,671 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:55:06,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:55:06,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:06,673 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:55:06,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:55:07,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:11,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:16,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:17,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:55:18,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:55:18,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:55:19,469 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:55:22,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:22,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:55:22,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:55:22,762 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:55:22,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:55:22,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:55:23,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:27,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:28,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:31,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:32,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:55:33,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:55:33,233 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:55:33,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:33,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:37,844 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:39,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:41,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:42,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:55:43,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:55:44,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:47,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:55:48,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:55:48,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:55:49,473 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:55:50,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:55,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:55:59,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:55:59,133 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:55:59,134 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:55:59,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:55:59,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:56:01,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:01,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:56:02,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:56:03,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:56:03,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:56:06,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:11,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:15,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:56:15,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:56:15,510 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:56:15,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:56:15,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:56:16,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:17,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:56:17,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:56:18,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:56:18,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:56:19,477 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:56:22,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:27,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:31,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:56:31,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:56:31,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:56:31,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:56:31,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:56:32,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:32,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:56:33,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:56:33,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:56:33,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:56:38,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:43,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:47,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:56:47,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:56:47,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:56:47,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:56:47,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:56:47,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:56:48,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:56:48,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:56:48,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:49,479 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:56:49,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:56:53,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:56:58,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:02,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:57:03,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:57:03,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:57:03,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:57:03,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:57:03,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:57:03,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:03,829 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:57:03,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:57:05,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:57:08,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:13,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:17,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:57:18,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:57:18,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:57:19,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:19,482 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:57:19,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:57:19,875 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:57:19,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:57:19,877 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:57:19,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:57:21,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:57:24,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:29,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:32,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:57:33,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:57:33,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:57:35,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:35,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:57:35,885 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:57:35,885 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:57:35,887 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:57:35,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:57:37,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:57:40,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:45,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:47,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:57:48,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:57:48,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:57:49,485 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:57:51,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:57:51,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:57:51,904 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:57:51,905 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:57:51,906 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:57:51,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:57:53,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:57:56,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:01,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:02,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:58:03,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:58:03,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:58:07,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:07,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:58:07,966 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:58:07,966 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:58:07,968 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:58:07,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:58:09,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:58:12,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:17,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:58:18,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:58:18,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:58:18,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:19,489 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:58:23,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:58:23,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:58:23,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:58:23,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:23,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:58:23,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:58:25,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:58:28,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:32,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:58:33,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:58:33,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:58:34,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:39,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:40,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:58:40,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:58:40,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:58:40,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:58:41,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:58:42,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:58:45,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:47,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:58:48,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:58:48,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:58:49,492 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:58:50,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:55,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:58:56,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:58:56,161 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:58:56,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:58:56,163 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:58:57,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:58:58,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:59:01,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:02,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:59:03,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:59:03,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:59:06,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:11,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:12,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:59:12,171 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:59:12,171 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:59:12,172 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:59:13,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:59:14,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:59:17,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:17,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:59:18,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:59:18,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:59:19,495 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:59:22,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:27,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:28,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:59:28,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:59:28,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:59:28,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:59:28,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:59:30,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:59:32,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:59:32,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:33,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:59:33,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:59:38,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:43,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:44,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 02:59:44,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 02:59:44,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 02:59:44,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 02:59:45,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 02:59:46,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 02:59:47,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 02:59:48,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 02:59:48,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 02:59:49,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:49,523 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 02:59:54,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 02:59:59,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:00,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:00:00,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:00:00,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:00:00,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:00:01,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:00:02,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:00:02,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:00:03,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:00:03,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:00:05,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:10,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:15,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:16,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:00:16,651 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:00:16,651 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:00:16,653 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:00:17,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:00:17,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:00:18,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:00:18,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:00:18,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:00:19,501 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:00:20,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:25,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:30,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:32,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:00:32,699 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:00:32,699 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:00:32,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:00:32,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:00:33,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:00:33,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:00:33,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:00:34,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:00:36,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:41,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:46,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:47,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:00:48,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:00:48,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:00:48,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:00:48,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:00:48,733 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:00:48,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:00:49,114 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:00:49,503 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:00:50,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:00:51,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:00:56,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:01,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:02,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:01:03,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:01:03,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:01:04,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:01:04,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:01:04,802 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:01:04,804 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:01:05,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:01:06,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:01:06,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:11,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:16,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:17,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:01:18,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:01:18,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:01:19,506 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:01:20,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:01:20,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:01:20,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:01:20,754 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:01:21,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:01:22,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:01:22,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:27,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:32,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:32,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:01:33,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:01:33,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:01:36,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:01:36,906 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:01:36,906 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:01:36,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:01:37,155 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:01:37,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:38,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:01:42,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:47,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:01:48,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:01:48,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:01:48,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:49,509 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:01:52,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:01:52,948 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:01:52,948 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:01:52,950 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:01:53,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:01:53,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:01:54,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:01:58,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:02,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:02:03,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:02:03,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:02:04,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:08,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:02:08,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:02:08,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:02:08,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:02:09,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:02:09,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:10,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:02:14,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:17,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:02:18,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:02:18,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:02:19,513 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:02:20,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:24,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:02:24,841 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:02:24,841 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:02:24,843 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:02:25,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:02:25,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:26,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:02:30,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:32,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:02:33,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:02:33,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:02:36,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:41,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:02:41,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:02:41,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:02:41,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:02:41,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:02:42,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:42,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:02:47,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:47,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:02:48,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:02:48,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:02:49,516 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:02:52,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:57,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:02:57,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:02:57,086 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:02:57,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:02:57,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:02:58,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:02:58,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:03:02,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:03:03,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:03:03,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:03:03,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:08,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:13,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:03:13,239 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:03:13,239 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:03:13,242 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:03:13,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:03:14,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:14,247 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:03:17,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:03:18,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:03:18,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:03:19,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:19,520 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:03:24,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:29,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:03:29,414 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:03:29,415 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:03:29,416 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:03:30,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:03:30,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:03:30,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:32,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:03:33,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:03:33,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:03:35,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:40,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:45,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:45,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:03:45,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:03:45,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:03:45,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:03:46,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:03:46,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:03:47,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:03:48,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:03:48,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:03:49,522 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:03:50,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:03:55,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:00,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:01,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:04:01,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:04:01,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:04:01,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:04:02,288 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:04:02,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:04:03,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:04:03,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:04:04,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:04:06,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:11,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:16,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:17,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:04:17,750 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:04:17,750 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:04:17,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:04:17,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:04:18,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:04:18,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:04:18,302 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:04:19,525 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:04:20,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:04:21,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:26,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:31,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:32,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:04:33,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:04:33,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:04:33,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:04:33,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:04:33,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:04:33,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:04:34,316 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:04:36,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:04:36,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:41,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:46,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:47,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:04:48,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:04:48,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:04:49,530 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:04:50,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:04:50,350 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:04:50,351 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:04:50,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:04:51,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:04:52,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:04:52,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:04:57,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:02,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:02,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:05:03,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:05:03,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:05:06,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:05:06,395 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:05:06,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:05:06,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:05:07,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:05:07,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:08,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:05:12,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:17,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:05:18,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:05:18,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:05:19,532 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:05:22,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:22,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:05:22,822 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:05:22,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:05:22,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:05:23,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:05:24,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:05:27,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:32,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:32,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:05:33,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:05:33,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:05:38,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:38,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:05:38,878 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:05:38,879 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:05:38,880 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:05:39,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:05:40,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:05:43,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:47,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:05:48,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:05:48,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:05:49,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:49,535 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:05:54,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:05:55,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:05:55,300 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:05:55,300 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:05:55,302 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:05:55,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:05:56,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:06:00,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:02,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:06:03,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:06:03,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:06:05,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:10,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:11,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:06:11,485 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:06:11,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:06:11,487 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:06:12,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:06:12,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:06:15,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:17,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:06:18,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:06:18,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:06:19,537 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:06:20,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:25,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:27,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:06:27,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:06:27,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:06:27,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:06:28,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:06:30,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:06:30,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:32,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:06:33,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:06:33,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:06:36,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:41,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:43,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:06:43,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:06:43,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:06:43,858 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:06:44,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:06:46,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:06:46,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:47,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:06:48,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:06:48,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:06:49,542 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:06:52,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:57,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:06:59,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:06:59,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:06:59,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:06:59,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:07:00,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:07:02,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:07:02,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:07:02,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:03,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:07:03,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:07:08,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:13,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:15,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:07:15,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:07:15,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:07:15,982 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:07:16,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:07:17,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:07:18,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:07:18,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:07:18,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:18,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:07:19,543 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:07:23,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:28,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:32,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:07:32,131 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:07:32,131 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:07:32,133 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:07:32,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:07:32,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:07:33,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:07:33,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:07:34,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:34,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:07:39,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:44,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:47,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:07:48,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:07:48,063 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:07:48,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:07:48,065 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:07:48,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:07:48,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:07:48,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:07:49,547 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:07:49,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:50,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:07:54,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:07:59,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:02,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:08:03,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:08:03,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:08:04,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:08:04,016 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:08:04,016 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:08:04,018 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:08:04,498 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:08:05,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:06,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:08:10,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:15,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:17,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:08:18,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:08:18,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:08:19,551 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:08:20,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:08:20,076 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:08:20,077 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:08:20,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:20,079 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:08:20,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:08:22,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:08:25,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:30,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:32,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:08:33,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:08:33,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:08:35,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:36,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:08:36,043 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:08:36,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:08:36,045 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:08:36,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:08:38,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:08:41,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:46,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:47,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:08:48,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:08:48,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:08:49,553 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:08:51,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:08:52,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:08:52,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:08:52,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:08:52,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:08:52,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:08:54,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:08:57,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:02,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:02,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:09:03,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:09:03,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:09:07,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:08,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:09:08,033 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:09:08,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:09:08,035 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:09:08,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:09:10,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:09:13,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:17,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:09:18,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:09:18,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:09:18,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:19,556 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:09:23,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:23,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:09:23,993 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:09:23,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:09:23,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:09:24,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:09:24,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:09:28,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:32,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:09:33,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:09:33,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:09:34,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:39,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:39,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:09:39,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:09:39,954 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:09:39,955 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:09:40,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:09:40,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:09:44,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:47,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:09:48,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:09:48,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:09:49,559 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:09:50,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:55,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:09:56,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:09:56,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:09:56,235 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:09:56,236 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:09:56,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:09:58,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:10:01,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:02,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:10:03,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:10:03,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:10:06,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:11,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:12,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:10:12,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:10:12,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:10:12,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:10:12,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:10:14,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:10:17,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:17,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:10:18,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:10:18,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:10:19,562 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:10:22,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:27,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:28,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:10:28,262 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:10:28,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:10:28,264 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:10:28,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:10:30,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:10:32,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:10:32,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:33,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:10:33,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:10:38,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:43,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:44,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:10:44,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:10:44,305 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:10:44,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:10:44,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:10:46,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:10:47,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:10:48,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:10:48,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:10:48,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:49,565 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:10:53,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:10:58,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:00,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:11:00,339 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:11:00,340 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:11:00,341 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:11:00,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:11:02,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:11:02,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:11:03,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:11:03,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:11:04,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:09,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:14,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:16,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:11:16,406 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:11:16,407 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:11:16,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:11:16,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:11:17,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:11:18,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:11:18,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:11:18,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:11:19,568 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:11:19,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:24,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:29,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:32,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:11:32,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:11:32,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:11:32,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:11:32,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:11:32,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:11:33,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:11:33,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:11:34,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:11:35,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:40,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:45,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:47,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:11:48,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:11:48,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:11:48,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:11:48,447 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:11:48,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:11:48,449 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:11:48,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:11:49,572 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:11:50,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:11:50,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:11:55,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:00,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:02,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:12:03,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:12:03,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:12:04,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:12:04,453 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:12:04,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:12:04,455 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:12:04,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:12:06,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:06,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:12:11,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:16,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:17,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:12:18,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:12:18,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:12:19,573 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:12:20,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:12:20,395 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:12:20,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:12:20,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:12:20,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:12:22,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:22,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:12:27,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:32,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:32,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:12:33,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:12:33,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:12:36,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:12:36,422 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:12:36,423 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:12:36,424 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:12:36,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:12:37,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:38,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:12:42,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:47,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:47,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:12:48,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:12:48,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:12:49,577 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:12:52,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:12:52,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:12:52,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:12:52,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:12:52,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:12:52,745 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:12:54,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:12:57,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:02,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:02,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:13:03,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:13:03,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:13:08,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:08,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:13:08,654 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:13:08,654 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:13:08,656 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:13:08,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:13:10,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:13:13,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:17,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:13:18,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:13:18,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:13:19,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:19,580 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:13:24,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:24,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:13:24,816 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:13:24,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:13:24,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:13:25,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:13:26,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:13:29,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:32,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:13:33,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:13:33,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:13:35,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:40,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:40,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:13:40,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:13:40,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:13:40,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:13:41,788 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:13:42,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:13:45,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:47,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:13:48,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:13:48,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:13:49,584 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:13:51,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:56,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:13:57,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:13:57,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:13:57,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:13:57,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:13:57,801 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:13:58,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:14:02,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:02,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:14:03,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:14:03,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:14:07,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:12,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:13,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:14:13,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:14:13,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:14:13,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:14:13,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:14:14,817 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:14:17,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:14:17,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:18,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:14:18,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:14:19,588 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:14:23,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:28,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:29,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:14:29,201 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:14:29,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:14:29,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:14:29,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:14:30,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:14:32,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:14:33,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:14:33,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:14:34,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:39,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:44,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:45,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:14:45,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:14:45,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:14:45,442 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:14:45,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:14:46,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:14:47,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:14:48,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:14:48,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:14:49,592 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:14:49,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:54,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:14:59,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:01,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:15:01,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:15:01,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:15:01,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:15:01,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:15:02,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:15:02,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:15:03,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:15:03,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:15:05,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:10,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:15,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:17,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:15:17,770 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:15:17,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:15:17,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:15:17,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:15:17,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:15:18,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:15:18,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:15:18,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:15:19,596 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:15:20,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:25,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:30,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:32,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:15:33,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:15:33,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:15:33,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:15:33,907 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:15:33,908 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:15:33,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:15:34,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:15:34,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:15:35,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:40,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:45,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:47,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:15:48,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:15:48,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:15:49,599 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:15:49,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:15:49,957 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:15:49,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:15:49,959 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:15:50,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:15:50,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:15:50,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:15:55,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:00,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:02,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:16:03,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:16:03,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:16:06,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:16:06,202 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:16:06,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:16:06,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:06,204 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:16:06,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:16:06,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:16:11,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:16,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:17,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:16:18,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:16:18,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:16:19,603 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:16:21,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:22,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:16:22,333 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:16:22,333 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:16:22,334 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:16:22,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:16:22,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:16:27,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:32,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:32,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:16:33,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:16:33,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:16:37,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:38,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:16:38,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:16:38,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:16:38,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:16:38,938 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:16:40,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:16:42,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:47,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:47,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:16:48,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:16:48,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:16:49,609 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:16:52,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:16:54,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:16:54,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:16:54,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:16:54,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:16:54,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:16:56,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:16:58,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:02,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:17:03,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:17:03,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:17:04,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:09,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:10,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:17:10,765 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:17:10,766 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:17:10,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:17:10,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:17:12,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:17:14,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:17,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:17:18,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:17:18,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:17:19,610 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:17:20,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:25,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:26,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:17:26,692 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:17:26,693 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:17:26,694 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:17:26,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:17:28,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:17:30,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:32,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:17:33,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:17:33,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:17:36,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:41,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:42,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:17:42,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:17:42,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:17:42,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:17:42,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:17:44,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:17:46,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:47,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:17:48,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:17:48,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:17:49,612 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:17:52,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:57,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:17:58,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:17:58,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:17:58,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:17:58,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:17:59,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:18:01,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:18:02,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:02,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:18:03,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:18:03,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:18:08,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:13,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:14,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:18:14,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:18:14,763 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:18:14,765 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:18:15,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:18:17,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:18:17,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:18:18,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:18:18,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:18:18,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:19,617 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:18:23,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:28,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:30,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:18:30,916 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:18:30,917 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:18:30,918 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:18:31,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:18:32,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:18:33,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:18:33,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:18:33,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:18:34,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:39,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:44,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:46,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:18:46,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:18:46,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:18:46,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:18:47,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:18:47,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:18:48,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:18:48,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:18:49,051 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:18:49,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:49,618 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:18:54,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:18:59,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:02,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:19:02,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:19:02,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:19:02,925 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:19:02,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:19:03,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:19:03,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:19:03,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:19:05,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:19:05,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:10,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:15,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:17,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:19:18,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:19:18,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:19:18,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:19:18,884 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:19:18,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:19:18,886 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:19:19,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:19:19,621 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:19:20,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:21,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:19:25,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:30,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:32,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:19:33,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:19:33,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:19:34,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:19:34,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:19:34,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:19:34,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:19:35,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:19:35,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:37,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:19:40,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:45,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:47,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:19:48,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:19:48,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:19:49,624 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:19:50,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:19:50,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:19:50,798 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:19:50,799 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:19:51,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:19:51,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:19:53,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:19:56,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:01,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:02,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:20:03,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:20:03,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:20:07,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:20:07,066 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:20:07,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:20:07,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:07,068 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:20:07,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:20:09,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:20:12,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:17,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:17,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:20:18,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:20:18,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:20:19,628 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:20:22,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:23,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:20:23,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:20:23,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:20:23,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:20:23,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:20:25,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:20:28,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:32,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:20:33,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:20:33,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:20:33,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:38,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:38,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:20:38,986 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:20:38,987 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:20:38,988 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:20:39,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:20:41,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:20:43,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:47,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:20:48,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:20:48,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:20:49,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:49,630 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:20:54,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:20:55,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:20:55,230 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:20:55,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:20:55,232 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:20:56,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:20:57,163 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:21:00,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:02,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:21:03,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:21:03,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:21:05,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:10,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:11,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:21:11,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:21:11,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:21:11,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:21:12,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:21:13,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:21:16,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:17,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:21:18,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:21:18,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:21:19,634 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:21:21,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:26,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:27,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:21:27,317 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:21:27,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:21:27,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:21:28,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:21:29,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:21:32,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:32,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:21:33,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:21:33,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:21:37,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:42,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:43,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:21:43,361 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:21:43,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:21:43,363 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:21:44,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:21:45,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:21:47,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:21:47,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:48,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:21:48,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:21:49,638 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:21:53,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:58,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:21:59,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:21:59,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:21:59,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:21:59,308 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:22:00,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:22:01,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:22:02,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:22:03,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:22:03,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:22:04,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:09,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:14,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:15,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:22:15,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:22:15,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:22:15,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:22:16,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:22:17,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:22:17,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:22:18,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:22:18,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:22:19,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:19,641 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:22:24,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:29,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:31,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:22:31,453 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:22:31,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:22:31,456 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:22:32,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:22:32,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:22:33,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:22:33,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:22:33,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:22:35,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:40,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:45,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:47,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:22:47,513 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:22:47,514 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:22:47,515 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:22:47,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:22:48,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:22:48,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:22:48,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:22:49,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:22:49,643 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:22:50,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:22:55,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:00,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:02,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:23:03,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:23:03,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:23:03,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:23:03,741 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:23:03,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:23:03,743 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:23:04,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:23:05,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:23:05,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:10,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:15,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:17,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:23:18,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:23:18,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:23:19,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:23:19,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:23:19,585 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:23:19,586 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:23:19,645 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:23:20,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:23:21,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:23:21,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:26,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:31,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:32,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:23:33,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:23:33,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:23:35,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:23:35,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:23:35,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:23:35,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:23:36,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:23:36,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:37,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:23:41,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:46,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:47,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:23:48,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:23:48,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:23:49,651 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:23:52,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:23:52,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:23:52,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:23:52,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:23:52,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:23:52,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:23:53,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:23:57,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:02,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:02,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:24:03,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:24:03,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:24:07,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:07,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:24:07,967 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:24:07,967 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:24:07,969 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:24:08,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:24:09,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:24:12,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:17,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:24:18,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:24:18,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:24:18,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:19,653 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:24:24,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:24:24,190 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:24:24,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:24:24,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:24,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:24:24,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:24:25,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:24:29,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:32,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:24:33,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:24:33,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:24:34,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:39,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:40,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:24:40,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:24:40,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:24:40,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:24:40,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:24:41,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:24:45,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:47,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:24:48,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:24:48,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:24:49,656 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:24:50,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:55,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:24:56,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:24:56,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:24:56,322 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:24:56,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:24:56,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:24:57,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:25:01,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:02,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:25:03,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:25:03,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:25:06,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:11,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:12,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:25:12,602 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:25:12,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:25:12,604 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:25:13,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:25:13,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:25:16,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:17,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:25:18,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:25:18,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:25:19,660 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:25:21,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:26,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:28,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:25:28,568 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:25:28,568 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:25:28,569 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:25:29,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:25:29,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:25:32,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:32,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:25:33,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:25:33,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:25:38,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:43,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:44,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:25:44,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:25:44,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:25:44,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:25:45,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:25:45,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:25:47,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:25:48,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:25:48,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:25:49,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:49,662 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:25:54,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:25:59,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:00,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:26:00,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:26:00,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:26:00,754 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:26:01,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:26:01,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:26:02,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:26:03,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:26:03,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:26:05,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:10,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:15,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:16,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:26:16,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:26:16,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:26:16,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:26:17,445 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:26:17,445 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:26:17,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:26:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:26:18,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:26:19,665 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:26:20,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:25,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:30,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:32,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:26:33,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:26:33,028 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:26:33,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:26:33,030 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:26:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:26:33,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:26:33,462 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:26:33,463 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:26:36,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:41,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:46,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:47,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:26:48,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:26:48,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:26:49,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:26:49,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:26:49,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:26:49,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:26:49,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:26:49,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:26:49,668 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:26:51,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:26:56,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:01,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:02,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:27:03,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:27:03,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:27:05,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:27:05,218 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:27:05,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:27:05,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:27:05,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:27:07,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:07,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:27:12,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:17,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:17,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:27:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:27:18,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:27:19,671 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:27:21,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:27:21,335 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:27:21,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:27:21,337 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:27:21,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:27:22,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:23,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:27:27,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:32,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:32,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:27:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:27:33,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:27:37,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:27:37,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:27:37,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:27:37,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:37,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:27:37,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:27:39,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:27:42,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:47,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:47,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:27:48,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:27:48,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:27:49,674 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:27:52,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:27:53,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:27:53,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:27:53,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:27:53,419 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:27:53,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:27:55,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:27:58,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:02,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:28:03,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:28:03,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:28:03,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:08,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:09,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:28:09,386 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:28:09,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:28:09,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:28:09,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:28:11,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:28:14,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:17,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:28:18,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:28:18,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:28:19,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:19,677 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:28:24,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:25,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:28:25,332 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:28:25,333 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:28:25,334 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:28:25,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:28:27,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:28:30,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:32,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:28:33,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:28:33,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:28:35,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:40,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:41,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:28:41,611 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:28:41,612 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:28:41,613 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:28:42,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:28:43,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:28:45,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:47,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:28:48,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:28:48,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:28:49,681 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:28:50,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:55,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:28:57,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:28:57,456 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:28:57,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:28:57,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:28:57,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:28:59,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:29:01,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:02,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:29:03,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:29:03,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:29:07,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:12,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:13,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:29:13,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:29:13,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:29:13,594 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:29:13,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:29:15,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:29:17,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:17,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:29:18,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:29:18,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:29:19,683 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:29:22,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:27,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:29,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:29:29,647 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:29:29,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:29:29,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:29:30,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:29:31,614 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:29:32,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:29:32,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:29:33,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:29:38,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:43,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:45,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:29:45,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:29:45,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:29:45,700 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:29:46,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:29:47,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:29:47,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:29:48,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:29:48,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:29:49,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:49,688 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:29:54,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:29:59,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:01,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:30:01,654 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:30:01,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:30:01,656 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:30:02,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:30:02,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:30:03,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:30:03,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:30:03,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:30:05,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:10,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:15,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:17,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:30:17,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:30:17,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:30:17,925 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:30:17,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:30:18,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:30:18,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:30:18,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:30:19,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:30:19,692 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:30:20,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:25,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:30,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:32,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:30:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:30:33,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:30:33,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:30:33,888 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:30:33,888 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:30:33,890 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:30:34,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:30:35,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:30:35,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:40,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:45,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:47,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:30:48,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:30:48,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:30:49,694 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:30:49,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:30:49,843 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:30:49,843 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:30:49,844 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:30:50,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:30:51,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:30:51,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:30:56,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:01,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:02,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:31:03,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:31:03,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:31:05,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:31:05,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:31:05,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:31:05,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:31:06,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:31:06,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:07,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:31:11,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:16,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:17,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:31:18,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:31:18,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:31:19,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:31:22,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:31:22,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:31:22,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:31:22,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:22,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:31:22,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:31:23,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:31:27,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:32,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:32,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:31:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:31:33,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:31:37,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:38,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:31:38,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:31:38,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:31:38,166 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:31:38,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:31:39,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:31:43,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:47,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:31:48,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:31:48,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:31:48,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:49,700 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:31:53,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:31:54,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:31:54,110 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:31:54,111 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:31:54,112 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:31:54,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:31:55,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:31:59,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:02,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:32:03,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:32:03,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:32:04,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:09,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:10,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:32:10,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:32:10,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:32:10,360 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:32:10,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:32:11,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:32:15,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:17,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:32:18,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:32:18,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:32:19,705 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:32:20,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:25,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:26,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:32:26,385 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:32:26,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:32:26,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:32:26,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:32:27,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:32:31,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:32,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:32:33,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:32:33,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:32:36,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:41,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:42,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:32:42,365 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:32:42,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:32:42,367 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:32:42,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:32:43,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:32:47,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:47,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:32:48,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:32:48,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:32:49,706 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:32:52,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:57,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:32:58,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:32:58,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:32:58,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:32:58,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:32:58,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:32:59,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:33:02,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:33:02,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:03,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:33:03,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:33:08,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:13,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:14,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:33:14,636 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:33:14,637 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:33:14,638 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:33:14,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:33:15,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:33:17,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:33:18,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:33:18,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:33:19,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:19,710 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:33:24,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:29,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:30,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:33:30,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:33:30,557 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:33:30,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:33:30,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:33:31,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:33:32,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:33:33,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:33:33,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:33:35,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:40,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:45,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:47,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:33:47,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:33:47,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:33:47,020 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:33:47,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:33:47,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:33:47,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:33:48,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:33:48,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:33:49,714 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:33:50,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:33:55,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:00,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:02,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:34:02,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:34:02,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:34:03,000 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:34:03,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:34:03,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:34:03,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:34:03,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:34:03,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:34:06,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:11,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:16,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:17,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:34:18,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:34:18,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:34:18,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:34:18,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:34:18,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:34:18,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:34:18,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:34:19,715 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:34:19,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:34:21,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:26,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:31,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:32,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:34:33,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:34:33,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:34:34,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:34:34,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:34:34,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:34:34,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:34:35,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:34:35,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:34:36,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:41,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:46,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:47,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:34:48,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:34:48,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:34:49,718 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:34:50,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:34:50,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:34:50,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:34:50,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:34:50,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:34:51,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:34:52,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:34:57,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:02,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:02,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:35:03,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:35:03,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:35:06,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:35:06,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:35:06,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:35:06,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:35:07,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:35:07,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:35:07,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:12,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:17,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:35:18,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:35:18,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:35:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:19,721 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:35:23,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:35:23,154 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:35:23,154 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:35:23,156 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:35:23,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:35:23,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:35:24,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:29,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:32,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:35:33,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:35:33,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:35:34,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:39,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:35:39,093 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:35:39,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:35:39,095 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:35:39,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:35:39,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:35:40,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:45,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:47,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:35:48,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:35:48,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:35:49,724 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:35:50,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:35:55,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:35:55,426 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:35:55,427 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:35:55,428 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:35:55,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:35:55,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:35:56,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:01,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:02,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:36:03,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:36:03,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:36:06,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:11,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:36:11,367 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:36:11,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:36:11,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:36:11,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:36:11,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:36:12,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:17,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:17,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:36:18,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:36:18,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:36:19,728 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:36:22,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:27,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:36:27,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:36:27,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:36:27,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:36:27,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:36:27,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:36:28,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:32,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:36:33,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:36:33,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:36:34,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:39,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:43,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:36:43,526 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:36:43,526 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:36:43,528 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:36:43,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:36:43,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:36:44,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:47,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:36:48,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:36:48,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:36:49,731 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:36:49,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:54,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:59,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:36:59,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:36:59,815 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:36:59,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:36:59,817 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:36:59,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:36:59,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:37:02,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:37:03,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:37:03,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:37:05,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:10,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:15,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:15,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:37:15,963 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:37:15,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:37:15,965 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:37:16,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:37:17,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:37:18,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:37:18,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:37:18,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:37:19,735 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:37:20,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:25,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:30,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:32,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:37:32,005 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:37:32,006 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:37:32,008 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:37:32,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:37:32,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:37:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:37:33,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:37:34,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:37:36,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:41,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:46,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:47,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:37:47,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:37:47,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:37:47,942 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:37:47,943 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:37:48,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:37:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:37:48,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:37:49,737 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:37:50,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:37:51,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:37:56,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:01,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:02,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:38:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:38:03,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:38:04,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:38:04,194 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:38:04,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:38:04,197 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:38:05,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:38:06,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:38:07,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:12,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:17,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:17,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:38:18,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:38:18,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:38:19,740 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:38:20,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:38:20,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:38:20,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:38:20,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:38:21,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:38:22,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:38:23,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:28,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:32,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:38:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:38:33,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:38:33,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:35,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:38:35,992 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:38:35,993 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:38:35,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:38:36,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:38:38,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:38:38,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:43,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:47,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:38:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:38:48,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:38:49,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:38:49,751 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:38:52,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:38:52,371 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:38:52,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:38:52,373 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:38:53,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:38:54,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:38:55,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:00,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:02,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:39:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:39:03,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:39:05,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:08,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:39:08,223 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:39:08,224 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:39:08,225 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:39:09,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:39:10,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:39:11,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:16,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:17,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:39:18,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:39:18,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:39:19,755 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:39:21,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:24,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:39:24,403 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:39:24,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:39:24,405 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:39:25,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:39:27,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:39:27,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:32,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:32,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:39:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:39:33,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:39:37,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:40,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:39:40,463 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:39:40,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:39:40,465 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:39:41,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:39:42,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:43,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:39:47,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:47,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:39:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:39:48,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:39:49,758 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:39:52,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:56,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:39:56,612 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:39:56,613 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:39:56,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:39:57,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:39:58,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:39:59,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:40:02,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:40:03,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:40:03,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:40:04,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:09,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:12,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:40:12,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:40:12,656 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:40:12,658 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:40:13,151 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:40:14,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:15,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:40:17,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:40:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:40:18,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:40:19,760 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:40:19,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:24,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:28,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:40:28,925 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:40:28,926 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:40:28,927 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:40:29,164 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:40:29,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:31,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:40:32,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:40:33,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:40:33,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:40:35,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:40,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:45,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:40:45,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:40:45,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:40:45,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:40:46,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:40:46,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:47,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:40:47,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:40:48,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:40:48,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:40:49,765 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:40:51,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:40:56,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:01,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:41:01,143 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:41:01,144 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:41:01,145 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:41:01,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:41:02,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:02,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:41:03,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:41:03,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:41:03,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:41:07,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:12,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:17,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:41:17,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:41:17,167 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:41:17,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:41:17,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:41:17,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:41:17,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:41:18,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:41:19,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:41:19,766 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:41:23,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:28,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:32,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:41:33,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:41:33,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:41:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:41:33,504 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:41:33,504 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:41:33,505 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:41:34,216 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:41:34,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:35,216 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:41:39,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:44,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:47,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:41:48,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:41:48,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:41:49,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:41:49,529 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:41:49,530 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:41:49,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:49,531 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:41:49,769 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:41:50,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:41:51,230 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:41:54,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:41:59,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:02,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:42:03,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:42:03,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:42:05,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:42:06,262 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:42:06,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:06,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:42:06,265 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:42:07,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:42:09,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:42:11,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:16,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:17,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:42:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:42:18,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:42:19,774 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:42:21,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:42:21,700 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:42:21,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:42:21,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:21,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:42:22,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:42:23,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:42:26,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:31,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:32,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:42:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:42:33,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:42:37,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:37,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:42:37,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:42:37,651 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:42:37,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:42:38,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:42:39,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:42:42,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:47,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:47,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:42:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:42:48,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:42:49,776 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:42:52,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:42:53,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:42:53,500 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:42:53,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:42:53,502 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:42:54,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:42:55,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:42:58,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:02,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:43:03,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:43:03,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:43:04,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:09,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:09,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:43:09,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:43:09,684 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:43:09,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:43:10,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:43:11,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:43:14,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:17,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:43:18,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:43:18,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:43:19,779 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:43:19,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:24,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:25,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:43:25,612 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:43:25,612 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:43:25,614 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:43:26,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:43:27,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:43:30,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:32,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:43:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:43:33,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:43:36,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:41,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:41,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:43:41,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:43:41,559 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:43:41,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:43:42,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:43:43,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:43:46,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:47,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:43:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:43:48,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:43:49,782 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:43:51,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:56,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:43:57,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:43:57,707 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:43:57,708 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:43:57,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:43:58,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:43:59,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:44:02,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:02,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:44:03,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:44:03,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:44:08,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:13,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:13,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:44:13,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:44:13,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:44:13,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:44:14,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:44:15,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:44:17,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:44:18,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:44:18,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:44:18,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:19,785 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:44:23,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:28,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:29,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:44:29,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:44:29,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:44:29,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:44:30,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:44:31,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:44:32,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:44:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:44:33,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:44:34,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:39,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:44,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:45,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:44:45,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:44:45,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:44:45,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:44:46,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:44:47,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:44:47,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:44:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:44:48,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:44:49,788 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:44:49,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:54,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:44:59,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:01,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:45:01,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:45:01,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:45:01,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:45:02,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:45:02,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:45:03,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:45:03,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:45:03,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:45:05,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:10,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:15,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:17,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:45:17,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:45:17,998 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:45:17,998 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:45:18,000 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:45:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:45:18,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:45:18,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:45:19,393 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:45:19,792 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:45:20,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:25,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:30,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:32,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:45:33,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:45:33,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:45:34,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:45:34,307 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:45:34,307 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:45:34,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:45:34,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:45:35,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:45:36,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:41,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:46,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:47,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:45:48,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:45:48,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:45:49,795 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:45:50,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:45:50,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:45:50,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:45:50,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:45:50,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:45:51,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:45:52,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:45:57,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:02,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:02,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:46:03,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:46:03,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:46:06,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:46:06,591 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:46:06,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:46:06,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:46:07,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:46:07,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:09,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:46:12,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:17,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:17,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:46:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:46:18,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:46:19,797 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:46:22,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:46:22,427 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:46:22,427 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:46:22,429 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:46:22,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:46:23,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:23,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:46:28,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:32,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:46:33,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:46:33,230 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:46:34,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:38,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:46:38,660 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:46:38,660 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:46:38,662 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:46:39,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:46:39,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:41,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:46:44,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:47,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:46:48,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:46:48,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:46:49,800 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:46:49,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:54,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:46:54,685 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:46:54,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:46:54,687 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:46:55,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:46:55,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:46:57,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:47:00,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:02,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:47:03,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:47:03,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:47:06,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:10,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:47:10,728 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:47:10,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:47:10,730 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:47:11,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:47:11,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:13,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:47:16,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:17,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:47:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:47:18,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:47:19,804 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:47:21,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:26,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:26,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:47:26,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:47:26,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:47:26,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:47:27,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:47:29,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:47:31,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:32,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:47:33,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:47:33,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:47:37,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:42,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:42,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:47:42,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:47:42,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:47:42,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:47:43,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:47:45,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:47:47,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:47,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:47:48,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:47:48,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:47:49,807 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:47:53,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:58,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:47:58,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:47:58,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:47:58,819 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:47:58,820 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:47:59,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:48:01,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:48:02,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:48:03,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:48:03,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:48:04,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:09,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:14,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:15,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:48:15,283 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:48:15,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:48:15,285 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:48:15,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:48:17,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:48:17,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:48:18,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:48:18,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:48:19,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:19,810 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:48:24,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:29,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:31,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:48:31,149 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:48:31,149 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:48:31,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:48:31,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:48:32,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:48:33,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:48:33,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:48:33,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:48:35,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:40,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:45,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:47,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:48:47,194 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:48:47,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:48:47,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:48:47,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:48:47,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:48:48,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:48:48,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:48:49,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:48:49,813 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:48:50,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:48:55,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:00,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:02,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:49:03,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:49:03,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:49:03,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:49:03,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:49:03,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:49:03,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:49:03,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:49:05,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:06,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:11,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:16,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:17,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:49:18,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:49:18,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:49:19,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:49:19,525 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:49:19,526 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:49:19,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:49:19,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:49:19,817 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:49:21,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:21,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:25,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:27,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:27,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:29,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:32,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:32,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:49:33,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:49:33,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:49:35,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:37,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:37,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:39,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:49:41,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:43,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:47,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:49:48,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:49:48,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:49:48,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:49,819 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:49:53,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:49:55,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:49:55,398 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:49:55,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:49:55,400 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:49:55,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:49:57,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:49:59,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:02,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:50:03,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:50:03,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:50:04,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:09,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:11,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:50:11,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:50:11,789 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:50:11,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:50:12,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:50:13,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:50:14,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:50:18,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:50:18,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:50:19,822 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:50:19,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:24,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:27,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:50:27,994 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:50:27,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:50:27,996 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:50:28,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:50:29,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:50:29,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:32,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:50:33,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:50:33,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:50:35,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:40,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:43,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:50:43,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:50:43,939 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:50:43,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:50:44,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:50:45,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:50:45,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:47,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:50:48,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:50:48,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:50:49,827 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:50:51,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:50:56,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:00,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:51:00,176 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:51:00,177 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:51:00,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:51:00,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:51:01,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:51:02,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:02,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:51:03,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:51:03,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:51:07,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:12,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:16,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:51:16,231 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:51:16,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:51:16,233 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:51:16,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:51:17,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:51:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:51:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:18,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:51:18,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:51:19,827 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:51:23,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:28,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:32,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:51:32,184 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:51:32,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:51:32,186 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:51:32,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:51:32,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:51:33,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:51:33,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:51:33,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:51:34,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:39,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:44,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:47,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:51:48,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:51:48,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:51:48,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:51:48,446 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:51:48,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:51:48,448 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:51:48,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:51:49,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:49,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:51:49,831 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:51:54,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:51:59,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:02,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:52:03,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:52:03,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:52:04,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:52:04,299 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:52:04,300 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:52:04,301 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:52:04,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:52:05,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:05,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:52:10,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:15,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:17,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:52:18,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:52:18,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:52:19,834 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:52:20,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:52:20,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:52:20,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:52:20,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:20,462 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:52:20,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:52:21,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:52:25,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:30,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:32,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:52:33,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:52:33,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:52:36,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:52:36,331 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:52:36,331 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:52:36,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:36,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:52:36,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:52:37,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:52:41,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:46,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:47,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:52:48,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:52:48,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:52:49,837 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:52:51,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:52:52,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:52:52,591 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:52:52,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:52:52,592 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:52:52,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:52:53,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:52:57,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:02,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:02,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:53:03,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:53:03,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:53:08,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:08,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:53:08,588 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:53:08,588 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:53:08,590 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:53:08,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:53:09,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:53:13,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:17,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:53:18,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:53:18,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:53:19,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:19,840 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:53:24,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:53:24,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:53:24,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:53:24,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:24,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:53:24,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:53:25,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:53:29,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:32,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:53:33,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:53:33,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:53:35,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:40,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:40,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:53:40,796 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:53:40,797 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:53:40,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:53:41,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:53:41,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:53:45,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:47,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:53:48,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:53:48,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:53:49,843 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:53:50,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:55,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:53:57,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:53:57,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:53:57,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:53:57,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:53:57,803 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:53:59,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:54:01,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:02,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:54:03,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:54:03,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:54:06,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:11,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:13,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:54:13,209 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:54:13,209 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:54:13,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:54:13,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:54:15,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:54:17,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:17,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:54:18,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:54:18,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:54:19,847 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:54:22,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:27,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:29,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:54:29,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:54:29,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:54:29,161 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:54:29,829 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:54:31,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:54:32,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:32,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:54:33,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:54:33,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:54:38,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:43,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:45,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:54:45,332 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:54:45,332 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:54:45,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:54:45,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:54:47,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:54:47,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:54:48,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:54:48,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:54:49,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:49,849 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:54:54,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:54:59,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:01,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:55:01,279 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:55:01,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:55:01,281 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:55:01,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:55:02,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:55:03,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:55:03,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:55:03,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:55:05,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:10,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:15,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:17,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:55:17,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:55:17,444 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:55:17,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:55:17,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:55:17,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:55:18,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:55:18,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:55:19,852 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:55:19,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:55:20,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:25,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:30,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:32,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:55:33,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:55:33,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:55:33,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:55:33,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:55:33,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:55:33,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:55:33,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:55:35,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:55:36,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:41,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:46,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:47,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:55:48,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:55:48,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:55:49,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:55:49,444 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:55:49,444 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:55:49,446 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:55:49,856 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:55:49,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:55:51,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:55:51,899 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:55:56,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:01,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:02,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:56:03,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:56:03,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:56:05,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:56:05,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:56:05,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:56:05,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:56:05,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:56:07,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:07,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:56:12,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:17,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:17,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:56:18,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:56:18,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:56:19,858 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:56:21,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:56:21,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:56:21,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:56:21,498 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:56:21,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:56:22,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:23,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:56:27,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:32,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:32,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:56:33,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:56:33,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:56:37,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:56:37,612 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:56:37,613 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:56:37,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:37,614 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:56:37,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:56:39,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:56:42,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:47,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:47,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:56:48,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:56:48,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:56:49,861 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:56:52,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:56:53,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:56:53,715 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:56:53,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:56:53,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:56:53,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:56:55,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:56:58,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:02,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:57:03,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:57:03,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:57:04,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:09,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:09,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:57:09,663 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:57:09,663 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:57:09,664 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:57:09,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:57:11,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:57:14,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:17,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:57:18,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:57:18,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:57:19,864 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:57:19,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:24,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:25,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:57:25,803 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:57:25,803 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:57:25,805 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:57:25,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:57:27,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:57:30,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:32,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:57:33,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:57:33,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:57:36,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:41,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:41,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:57:41,840 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:57:41,841 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:57:41,842 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:57:41,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:57:43,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:57:46,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:47,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:57:48,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:57:48,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:57:49,869 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:57:51,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:56,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:57:57,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:57:57,984 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:57:57,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:57:57,986 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:57:58,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:58:00,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:58:01,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:02,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:58:03,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:58:03,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:58:07,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:12,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:13,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:58:13,936 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:58:13,936 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:58:13,938 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:58:14,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:58:16,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:58:17,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:17,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:58:18,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:58:18,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:58:19,870 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:58:23,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:28,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:29,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:58:29,966 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:58:29,967 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:58:29,968 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:58:30,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:58:32,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:58:32,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:58:33,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:58:33,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:58:34,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:39,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:44,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:46,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:58:46,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:58:46,125 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:58:46,127 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:58:47,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:58:47,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:58:48,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:58:48,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:58:48,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:58:49,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:49,903 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:58:54,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:58:59,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:02,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:59:02,277 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:59:02,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:59:02,279 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:59:02,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:59:03,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:59:03,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:59:03,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:59:04,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:59:05,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:10,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:15,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:17,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:59:18,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:59:18,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:59:18,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:59:18,518 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:59:18,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:59:18,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:59:19,069 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:59:19,907 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:59:20,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:59:20,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:25,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:30,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:32,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:59:33,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:59:33,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:59:34,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:59:34,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:59:34,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:59:34,382 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:59:35,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:59:36,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:59:36,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:41,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:46,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:47,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 03:59:48,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 03:59:48,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 03:59:49,909 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 03:59:50,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 03:59:50,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 03:59:50,444 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 03:59:50,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 03:59:51,095 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 03:59:51,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 03:59:52,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 03:59:56,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:01,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:02,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:00:03,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:00:03,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:00:06,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:00:06,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:00:06,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:00:06,396 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:00:07,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:00:07,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:08,109 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:00:12,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:17,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:17,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:00:18,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:00:18,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:00:19,912 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:00:22,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:00:22,830 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:00:22,830 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:00:22,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:22,832 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:00:23,122 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:00:24,122 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:00:27,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:32,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:32,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:00:33,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:00:33,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:00:38,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:38,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:00:38,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:00:38,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:00:38,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:00:39,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:00:40,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:00:43,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:47,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:00:48,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:00:48,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:00:49,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:49,915 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:00:54,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:00:54,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:00:54,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:00:54,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:00:54,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:00:55,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:00:56,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:00:59,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:02,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:01:03,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:01:03,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:01:05,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:10,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:10,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:01:10,993 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:01:10,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:01:10,995 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:01:11,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:01:12,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:01:15,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:17,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:01:18,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:01:18,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:01:19,918 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:01:21,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:26,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:27,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:01:27,222 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:01:27,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:01:27,224 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:01:28,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:01:28,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:01:32,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:32,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:01:33,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:01:33,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:01:37,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:42,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:43,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:01:43,173 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:01:43,173 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:01:43,175 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:01:43,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:01:44,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:01:47,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:47,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:01:48,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:01:48,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:01:49,921 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:01:52,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:57,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:01:59,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:01:59,321 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:01:59,322 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:01:59,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:02:00,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:02:00,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:02:02,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:02:03,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:02:03,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:02:03,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:08,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:13,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:15,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:02:15,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:02:15,400 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:02:15,401 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:02:16,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:02:16,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:02:17,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:02:18,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:02:18,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:02:18,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:19,925 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:02:23,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:28,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:31,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:02:31,351 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:02:31,351 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:02:31,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:02:32,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:02:32,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:02:32,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:02:33,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:02:33,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:02:34,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:39,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:44,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:47,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:02:47,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:02:47,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:02:47,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:02:47,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:02:48,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:02:48,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:02:48,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:02:48,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:02:49,928 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:02:49,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:54,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:02:59,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:02,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:03:03,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:03:03,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:03:03,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:03:03,663 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:03:03,664 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:03:03,665 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:03:04,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:03:05,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:06,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:03:10,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:15,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:17,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:03:18,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:03:18,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:03:19,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:03:19,501 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:03:19,501 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:03:19,503 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:03:19,930 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:03:20,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:03:20,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:03:20,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:25,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:30,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:32,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:03:33,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:03:33,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:03:35,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:03:35,503 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:03:35,504 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:03:35,506 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:03:36,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:03:36,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:03:36,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:41,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:46,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:47,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:03:48,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:03:48,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:03:49,933 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:03:51,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:03:51,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:03:51,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:03:51,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:03:51,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:03:52,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:03:52,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:03:56,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:01,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:02,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:04:03,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:04:03,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:04:07,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:07,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:04:07,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:04:07,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:04:07,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:04:08,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:04:10,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:04:12,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:17,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:04:18,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:04:18,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:04:18,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:19,936 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:04:23,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:23,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:04:23,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:04:23,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:04:23,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:04:24,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:04:26,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:04:28,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:32,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:04:33,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:04:33,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:04:34,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:39,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:39,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:04:39,925 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:04:39,926 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:04:39,927 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:04:40,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:04:42,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:04:44,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:47,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:04:48,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:04:48,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:04:49,939 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:04:49,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:54,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:04:55,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:04:55,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:04:55,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:04:55,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:04:56,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:04:58,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:04:59,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:02,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:05:03,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:05:03,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:05:05,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:10,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:12,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:05:12,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:05:12,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:05:12,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:05:12,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:05:14,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:05:16,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:17,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:05:18,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:05:18,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:05:19,943 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:05:21,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:26,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:28,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:05:28,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:05:28,244 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:05:28,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:05:28,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:05:30,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:05:32,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:32,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:05:33,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:05:33,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:05:37,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:42,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:44,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:05:44,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:05:44,543 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:05:44,545 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:05:45,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:05:46,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:05:47,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:47,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:05:48,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:05:48,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:05:49,946 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:05:52,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:05:57,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:00,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:06:00,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:06:00,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:06:00,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:06:01,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:06:02,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:06:02,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:06:03,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:06:03,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:06:03,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:08,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:13,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:16,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:06:16,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:06:16,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:06:16,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:06:17,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:06:17,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:06:18,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:06:18,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:06:18,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:06:18,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:19,950 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:06:23,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:28,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:32,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:06:32,737 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:06:32,737 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:06:32,739 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:06:32,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:06:33,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:06:33,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:06:33,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:06:34,422 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:06:34,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:39,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:44,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:47,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:06:48,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:06:48,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:06:48,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:06:48,876 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:06:48,877 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:06:48,878 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:06:49,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:06:49,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:49,952 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:06:50,435 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:06:54,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:06:59,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:02,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:07:03,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:07:03,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:07:04,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:07:04,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:07:04,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:07:04,828 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:07:05,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:07:05,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:06,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:07:10,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:15,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:17,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:07:18,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:07:18,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:07:19,955 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:07:20,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:07:20,872 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:07:20,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:07:20,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:20,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:07:21,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:07:22,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:07:25,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:30,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:32,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:07:33,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:07:33,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:07:36,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:36,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:07:36,901 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:07:36,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:07:36,903 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:07:37,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:07:38,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:07:41,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:46,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:47,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:07:48,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:07:48,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:07:49,959 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:07:51,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:07:52,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:07:52,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:07:52,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:07:52,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:07:53,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:07:54,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:07:57,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:02,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:08:03,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:08:03,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:08:03,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:08,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:09,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:08:09,086 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:08:09,086 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:08:09,088 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:08:09,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:08:10,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:08:14,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:17,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:08:18,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:08:18,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:08:19,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:19,960 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:08:24,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:25,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:08:25,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:08:25,142 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:08:25,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:08:25,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:08:26,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:08:30,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:32,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:08:33,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:08:33,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:08:35,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:40,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:41,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:08:41,194 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:08:41,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:08:41,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:08:41,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:08:42,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:08:46,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:47,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:08:48,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:08:48,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:08:49,963 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:08:51,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:56,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:08:57,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:08:57,359 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:08:57,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:08:57,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:08:57,534 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:08:58,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:09:02,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:02,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:09:03,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:09:03,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:09:07,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:12,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:13,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:09:13,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:09:13,400 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:09:13,401 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:09:13,547 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:09:14,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:09:17,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:17,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:09:18,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:09:18,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:09:19,967 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:09:22,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:27,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:29,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:09:29,539 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:09:29,539 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:09:29,541 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:09:29,559 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:09:30,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:09:32,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:09:33,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:09:33,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:09:33,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:38,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:43,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:45,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:09:45,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:09:45,558 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:09:45,560 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:09:45,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:09:46,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:09:47,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:09:48,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:09:48,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:09:49,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:49,970 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:09:54,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:09:59,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:01,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:10:01,772 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:10:01,772 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:10:01,773 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:10:02,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:10:02,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:10:02,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:10:03,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:10:03,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:10:05,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:10,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:15,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:17,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:10:17,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:10:17,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:10:17,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:10:17,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:10:18,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:10:18,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:10:18,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:10:18,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:10:19,973 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:10:20,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:25,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:30,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:32,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:10:33,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:10:33,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:10:34,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:10:34,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:10:34,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:10:34,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:10:34,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:10:34,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:10:36,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:41,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:46,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:47,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:10:48,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:10:48,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:10:49,976 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:10:50,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:10:50,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:10:50,322 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:10:50,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:10:50,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:10:51,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:10:52,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:10:56,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:01,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:02,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:11:03,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:11:03,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:11:06,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:11:06,272 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:11:06,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:11:06,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:11:06,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:11:07,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:08,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:11:12,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:17,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:17,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:11:18,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:11:18,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:11:19,980 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:11:22,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:11:22,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:11:22,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:11:22,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:22,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:11:22,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:11:24,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:11:27,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:32,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:32,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:11:33,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:11:33,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:11:37,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:38,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:11:38,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:11:38,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:11:38,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:11:39,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:11:40,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:11:42,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:47,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:47,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:11:48,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:11:48,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:11:49,982 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:11:52,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:11:54,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:11:54,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:11:54,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:11:54,645 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:11:54,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:11:56,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:11:58,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:02,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:12:03,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:12:03,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:12:04,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:09,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:10,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:12:10,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:12:10,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:12:10,584 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:12:10,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:12:12,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:12:14,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:17,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:12:18,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:12:18,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:12:19,985 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:12:19,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:24,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:26,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:12:26,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:12:26,665 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:12:26,666 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:12:26,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:12:28,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:12:30,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:32,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:12:33,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:12:33,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:12:36,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:41,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:42,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:12:42,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:12:42,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:12:42,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:12:42,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:12:44,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:12:46,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:47,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:12:48,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:12:48,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:12:49,988 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:12:51,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:56,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:12:58,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:12:58,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:12:58,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:12:58,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:12:58,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:13:00,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:13:02,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:02,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:13:03,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:13:03,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:13:08,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:13,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:14,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:13:14,645 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:13:14,645 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:13:14,647 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:13:14,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:13:16,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:13:17,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:13:18,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:13:18,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:13:18,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:19,991 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:13:23,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:28,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:30,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:13:30,685 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:13:30,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:13:30,687 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:13:30,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:13:32,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:13:32,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:13:33,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:13:33,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:13:34,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:39,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:44,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:46,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:13:46,626 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:13:46,626 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:13:46,628 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:13:46,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:13:47,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:13:48,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:13:48,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:13:48,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:13:49,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:49,994 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:13:54,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:13:59,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:02,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:14:03,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:14:03,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:14:03,006 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:14:03,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:14:03,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:14:03,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:14:03,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:14:04,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:14:05,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:10,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:15,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:17,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:14:18,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:14:18,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:14:19,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:14:19,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:14:19,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:14:19,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:14:19,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:14:19,996 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:14:20,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:14:20,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:25,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:30,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:32,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:14:33,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:14:33,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:14:35,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:14:35,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:14:35,234 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:14:35,236 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:14:35,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:14:36,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:36,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:14:41,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:46,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:47,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:14:48,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:14:48,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:14:50,001 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:14:51,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:14:51,213 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:14:51,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:14:51,215 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:14:51,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:14:52,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:14:52,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:14:57,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:02,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:02,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:15:03,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:15:03,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:15:07,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:15:07,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:15:07,422 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:15:07,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:07,423 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:15:07,831 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:15:08,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:15:12,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:17,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:17,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:15:18,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:15:18,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:15:20,003 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:15:23,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:23,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:15:23,498 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:15:23,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:15:23,500 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:15:23,844 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:15:24,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:15:28,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:32,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:15:33,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:15:33,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:15:33,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:38,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:39,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:15:39,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:15:39,540 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:15:39,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:15:39,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:15:40,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:15:43,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:47,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:15:48,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:15:48,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:15:49,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:50,006 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:15:55,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:15:55,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:15:55,867 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:15:55,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:15:55,870 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:15:55,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:15:56,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:16:00,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:02,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:16:03,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:16:03,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:16:06,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:11,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:11,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:16:11,912 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:16:11,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:16:11,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:16:12,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:16:12,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:16:16,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:17,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:16:18,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:16:18,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:16:20,010 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:16:22,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:27,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:27,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:16:27,847 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:16:27,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:16:27,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:16:27,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:16:28,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:16:32,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:32,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:16:33,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:16:33,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:16:38,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:43,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:44,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:16:44,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:16:44,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:16:44,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:16:44,910 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:16:44,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:16:47,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:16:48,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:16:48,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:16:48,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:50,012 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:16:54,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:16:59,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:00,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:17:00,198 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:17:00,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:17:00,200 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:17:00,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:17:00,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:17:02,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:17:03,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:17:03,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:17:04,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:09,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:14,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:16,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:17:16,263 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:17:16,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:17:16,265 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:17:16,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:17:16,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:17:17,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:17:18,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:17:18,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:17:20,015 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:17:20,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:25,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:30,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:32,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:17:32,513 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:17:32,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:17:32,515 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:17:32,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:17:32,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:17:32,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:17:33,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:17:33,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:17:35,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:40,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:45,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:47,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:17:48,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:17:48,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:17:48,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:17:48,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:17:48,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:17:48,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:17:48,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:17:48,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:17:50,018 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:17:51,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:17:56,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:01,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:02,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:18:03,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:18:03,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:18:04,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:18:04,562 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:18:04,562 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:18:04,564 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:18:04,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:18:04,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:18:06,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:11,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:16,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:17,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:18:18,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:18:18,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:18:20,022 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:18:20,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:18:20,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:18:20,508 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:18:20,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:18:20,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:18:20,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:18:22,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:27,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:32,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:32,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:18:33,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:18:33,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:18:36,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:18:36,652 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:18:36,653 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:18:36,654 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:18:37,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:18:37,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:39,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:18:42,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:47,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:47,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:18:48,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:18:48,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:18:50,024 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:18:52,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:18:52,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:18:52,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:18:52,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:18:52,683 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:18:53,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:18:55,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:18:57,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:02,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:02,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:19:03,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:19:03,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:19:08,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:08,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:19:08,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:19:08,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:19:08,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:19:09,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:19:11,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:19:13,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:17,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:19:18,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:19:18,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:19:19,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:20,027 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:19:25,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:25,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:19:25,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:19:25,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:19:25,256 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:19:26,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:19:27,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:19:30,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:32,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:19:33,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:19:33,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:19:35,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:40,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:41,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:19:41,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:19:41,409 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:19:41,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:19:42,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:19:43,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:19:46,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:47,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:19:48,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:19:48,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:19:50,030 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:19:52,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:57,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:19:57,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:19:57,524 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:19:57,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:19:57,526 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:19:58,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:19:59,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:20:02,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:02,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:20:03,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:20:03,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:20:07,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:12,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:13,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:20:13,841 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:20:13,842 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:20:13,844 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:20:14,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:20:15,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:20:17,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:17,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:20:18,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:20:18,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:20:20,033 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:20:23,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:28,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:29,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:20:29,799 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:20:29,799 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:20:29,800 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:20:30,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:20:31,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:20:32,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:20:33,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:20:33,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:20:33,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:38,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:43,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:45,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:20:45,920 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:20:45,921 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:20:45,922 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:20:46,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:20:47,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:20:47,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:20:48,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:20:48,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:20:49,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:20:50,036 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:20:55,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:00,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:02,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:21:02,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:21:02,145 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:21:02,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:21:02,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:21:03,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:21:03,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:21:03,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:21:03,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:21:05,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:10,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:15,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:17,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:21:18,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:21:18,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:21:18,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:21:18,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:21:18,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:21:18,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:21:18,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:21:19,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:21:20,040 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:21:21,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:26,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:31,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:32,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:21:33,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:21:33,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:21:34,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:21:34,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:21:34,236 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:21:34,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:21:35,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:21:35,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:21:36,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:41,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:46,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:47,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:21:48,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:21:48,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:21:50,043 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:21:50,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:21:50,343 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:21:50,343 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:21:50,345 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:21:51,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:21:51,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:21:51,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:21:56,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:01,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:02,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:22:03,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:22:03,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:22:06,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:22:06,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:22:06,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:22:06,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:22:07,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:22:07,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:22:07,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:12,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:17,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:17,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:22:18,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:22:18,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:22:20,046 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:22:22,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:22:22,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:22:22,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:22:22,244 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:22:23,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:22:23,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:22:23,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:28,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:32,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:22:33,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:22:33,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:22:33,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:38,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:38,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:22:38,512 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:22:38,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:22:38,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:22:39,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:22:39,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:22:43,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:47,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:22:48,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:22:48,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:22:48,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:50,049 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:22:54,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:22:54,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:22:54,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:22:54,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:22:54,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:22:55,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:22:55,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:22:59,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:02,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:23:03,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:23:03,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:23:05,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:10,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:23:10,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:23:10,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:23:10,412 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:23:11,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:23:11,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:23:11,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:16,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:17,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:23:18,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:23:18,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:23:20,053 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:23:22,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:26,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:23:26,458 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:23:26,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:23:26,460 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:23:27,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:23:27,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:23:27,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:32,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:32,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:23:33,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:23:33,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:23:38,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:42,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:23:42,621 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:23:42,621 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:23:42,623 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:23:43,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:23:43,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:23:43,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:47,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:23:48,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:23:48,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:23:49,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:50,055 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:23:55,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:23:58,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:23:58,682 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:23:58,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:23:58,684 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:23:59,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:23:59,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:24:00,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:02,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:24:03,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:24:03,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:24:06,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:11,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:14,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:24:14,740 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:24:14,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:24:14,742 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:24:15,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:24:15,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:24:16,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:17,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:24:18,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:24:18,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:24:20,060 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:24:22,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:27,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:30,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:24:30,795 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:24:30,795 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:24:30,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:24:31,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:24:31,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:24:32,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:32,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:24:33,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:24:33,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:24:38,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:43,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:46,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:24:46,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:24:46,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:24:46,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:24:47,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:24:47,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:24:47,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:24:48,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:24:48,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:24:49,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:24:50,064 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:24:55,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:00,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:02,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:25:02,689 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:25:02,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:25:02,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:25:02,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:25:03,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:25:03,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:25:03,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:25:03,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:25:05,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:10,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:15,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:17,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:25:18,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:25:18,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:25:19,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:25:19,056 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:25:19,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:25:19,058 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:25:19,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:25:19,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:25:20,065 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:25:21,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:26,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:31,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:32,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:25:33,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:25:33,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:25:34,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:25:34,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:25:34,923 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:25:34,924 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:25:35,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:25:35,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:25:36,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:41,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:46,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:47,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:25:48,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:25:48,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:25:50,069 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:25:50,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:25:50,978 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:25:50,978 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:25:50,980 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:25:51,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:25:51,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:25:51,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:25:56,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:01,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:02,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:26:03,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:26:03,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:26:07,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:26:07,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:26:07,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:26:07,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:07,433 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:26:08,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:26:09,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:26:12,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:17,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:17,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:26:18,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:26:18,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:26:20,072 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:26:23,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:23,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:26:23,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:26:23,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:26:23,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:26:24,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:26:25,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:26:28,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:32,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:26:33,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:26:33,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:26:33,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:38,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:39,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:26:39,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:26:39,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:26:39,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:26:39,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:26:41,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:26:44,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:47,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:26:48,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:26:48,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:26:49,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:50,076 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:26:55,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:26:55,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:26:55,761 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:26:55,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:26:55,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:26:56,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:26:57,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:27:00,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:02,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:27:03,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:27:03,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:27:06,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:11,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:11,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:27:11,799 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:27:11,799 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:27:11,801 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:27:12,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:27:13,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:27:16,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:17,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:27:18,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:27:18,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:27:20,078 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:27:22,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:27,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:27,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:27:27,862 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:27:27,862 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:27:27,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:27:28,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:27:29,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:27:32,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:32,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:27:33,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:27:33,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:27:38,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:43,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:44,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:27:44,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:27:44,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:27:44,104 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:27:44,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:27:45,470 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:27:47,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:27:48,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:27:48,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:27:49,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:27:50,081 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:27:55,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:00,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:28:00,063 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:28:00,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:28:00,065 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:28:00,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:28:01,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:01,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:28:02,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:28:03,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:28:03,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:28:06,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:11,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:16,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:28:16,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:28:16,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:28:16,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:28:16,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:28:17,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:17,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:28:17,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:28:18,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:28:18,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:28:20,085 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:28:23,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:28,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:32,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:28:32,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:28:32,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:28:32,304 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:28:32,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:28:32,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:28:33,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:28:33,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:28:33,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:33,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:28:38,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:43,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:47,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:28:48,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:28:48,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:28:48,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:48,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:28:48,573 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:28:48,573 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:28:48,575 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:28:49,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:28:49,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:28:50,087 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:28:54,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:28:59,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:02,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:29:03,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:29:03,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:29:04,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:04,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:29:04,623 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:29:04,623 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:29:04,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:29:05,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:29:05,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:29:09,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:14,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:17,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:29:18,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:29:18,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:29:20,091 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:29:20,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:20,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:29:20,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:29:20,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:29:20,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:29:21,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:29:21,552 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:29:25,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:30,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:32,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:29:33,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:29:33,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:29:36,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:37,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:29:37,284 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:29:37,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:29:37,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:29:37,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:29:37,565 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:29:42,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:47,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:47,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:29:48,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:29:48,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:29:50,093 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:29:53,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:29:53,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:29:53,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:29:53,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:29:53,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:29:53,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:29:53,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:29:58,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:02,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:30:03,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:30:03,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:30:03,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:08,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:09,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:30:09,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:30:09,442 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:30:09,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:30:09,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:30:11,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:30:14,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:17,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:30:18,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:30:18,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:30:19,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:20,096 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:30:25,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:25,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:30:25,743 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:30:25,743 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:30:25,745 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:30:26,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:30:27,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:30:30,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:32,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:30:33,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:30:33,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:30:36,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:41,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:41,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:30:41,830 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:30:41,830 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:30:41,832 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:30:42,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:30:43,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:30:46,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:47,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:30:48,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:30:48,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:30:50,098 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:30:52,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:57,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:30:57,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:30:57,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:30:57,850 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:30:57,851 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:30:58,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:30:59,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:31:02,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:02,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:31:03,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:31:03,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:31:08,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:13,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:14,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:31:14,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:31:14,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:31:14,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:31:14,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:31:15,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:31:17,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:31:18,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:31:18,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:31:19,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:20,102 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:31:25,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:29,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:31:29,899 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:31:29,900 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:31:29,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:31:30,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:31:30,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:31,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:31:32,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:31:33,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:31:33,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:31:36,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:41,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:45,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:31:45,900 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:31:45,900 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:31:45,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:31:46,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:31:46,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:47,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:31:47,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:31:48,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:31:48,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:31:50,106 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:31:52,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:31:57,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:01,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:32:01,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:32:01,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:32:01,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:32:02,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:32:02,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:02,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:32:03,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:32:03,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:32:03,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:32:08,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:13,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:17,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:32:17,848 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:32:17,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:32:17,850 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:32:17,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:32:18,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:32:18,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:32:18,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:32:19,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:20,108 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:32:20,699 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:32:25,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:30,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:32,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:32:33,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:32:33,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:32:33,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:32:33,890 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:32:33,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:32:33,892 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:32:34,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:32:35,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:36,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:32:40,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:45,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:47,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:32:48,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:32:48,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:32:50,111 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:32:50,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:32:50,152 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:32:50,152 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:32:50,154 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:32:50,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:32:51,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:32:52,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:32:56,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:01,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:02,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:33:03,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:33:03,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:33:06,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:33:06,211 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:33:06,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:33:06,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:06,213 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:33:06,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:33:08,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:33:11,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:16,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:17,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:33:18,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:33:18,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:33:20,114 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:33:22,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:33:22,047 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:33:22,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:33:22,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:22,049 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:33:22,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:33:24,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:33:27,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:32,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:32,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:33:33,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:33:33,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:33:37,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:38,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:33:38,181 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:33:38,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:33:38,183 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:33:38,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:33:40,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:33:43,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:47,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:33:48,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:33:48,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:33:48,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:50,119 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:33:54,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:33:54,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:33:54,329 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:33:54,330 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:33:54,331 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:33:54,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:33:56,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:33:59,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:02,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:34:03,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:34:03,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:34:04,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:09,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:10,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:34:10,388 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:34:10,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:34:10,390 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:34:10,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:34:12,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:34:15,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:17,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:34:18,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:34:18,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:34:20,122 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:34:21,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:26,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:26,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:34:26,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:34:26,348 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:34:26,349 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:34:26,803 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:34:28,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:34:31,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:32,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:34:33,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:34:33,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:34:36,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:41,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:42,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:34:42,499 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:34:42,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:34:42,501 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:34:42,817 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:34:44,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:34:46,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:47,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:34:48,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:34:48,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:34:50,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:34:52,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:57,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:34:58,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:34:58,547 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:34:58,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:34:58,549 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:34:58,829 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:35:00,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:35:02,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:02,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:35:03,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:35:03,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:35:08,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:13,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:14,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:35:14,389 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:35:14,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:35:14,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:35:14,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:35:16,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:35:17,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:35:18,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:35:18,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:35:19,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:20,129 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:35:25,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:30,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:30,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:35:30,647 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:35:30,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:35:30,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:35:30,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:35:32,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:35:32,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:35:33,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:35:33,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:35:36,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:41,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:46,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:46,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:35:46,495 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:35:46,495 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:35:46,497 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:35:46,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:35:47,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:35:48,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:35:48,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:35:48,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:35:50,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:35:51,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:35:56,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:01,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:02,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:36:02,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:36:02,559 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:36:02,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:36:02,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:36:02,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:36:03,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:36:03,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:36:04,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:36:06,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:11,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:16,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:17,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:36:18,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:36:18,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:36:19,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:36:19,013 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:36:19,013 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:36:19,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:36:19,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:36:20,133 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:36:20,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:36:22,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:27,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:32,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:32,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:36:33,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:36:33,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:36:34,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:36:34,950 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:36:34,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:36:34,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:36:35,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:36:36,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:36:37,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:42,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:47,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:36:48,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:36:48,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:36:48,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:50,136 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:36:51,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:36:51,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:36:51,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:36:51,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:36:51,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:36:52,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:36:54,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:36:59,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:02,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:37:03,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:37:03,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:37:04,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:07,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:37:07,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:37:07,248 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:37:07,249 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:37:07,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:37:08,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:37:10,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:15,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:17,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:37:18,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:37:18,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:37:20,140 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:37:21,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:23,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:37:23,474 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:37:23,474 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:37:23,476 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:37:23,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:37:24,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:37:26,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:31,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:32,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:37:33,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:37:33,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:37:36,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:39,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:37:39,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:37:39,438 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:37:39,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:37:39,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:37:40,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:37:42,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:47,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:47,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:37:48,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:37:48,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:37:50,143 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:37:53,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:37:55,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:37:55,676 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:37:55,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:37:55,678 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:37:55,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:37:56,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:37:58,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:02,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:38:03,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:38:03,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:38:04,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:09,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:11,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:38:11,855 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:38:11,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:38:11,857 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:38:11,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:38:12,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:38:14,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:17,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:38:18,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:38:18,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:38:20,146 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:38:20,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:25,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:28,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:38:28,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:38:28,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:38:28,181 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:38:29,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:38:30,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:31,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:38:32,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:38:33,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:38:33,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:38:35,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:40,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:44,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:38:44,118 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:38:44,118 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:38:44,120 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:38:45,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:38:46,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:47,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:38:47,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:38:48,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:38:48,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:38:50,149 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:38:51,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:38:56,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:00,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:39:00,459 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:39:00,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:39:00,461 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:39:01,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:39:01,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:02,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:39:03,028 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:39:03,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:39:03,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:39:06,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:11,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:16,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:39:16,501 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:39:16,501 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:39:16,503 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:39:16,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:17,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:39:17,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:39:18,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:39:18,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:39:19,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:39:20,152 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:39:22,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:27,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:32,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:32,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:39:32,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:39:32,805 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:39:32,807 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:39:32,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:39:33,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:39:33,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:39:33,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:39:35,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:39:37,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:42,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:47,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:47,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:39:48,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:39:48,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:39:49,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:39:49,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:39:49,143 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:39:49,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:39:50,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:39:50,156 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:39:51,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:39:53,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:39:58,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:02,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:40:03,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:40:03,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:40:03,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:05,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:40:05,087 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:40:05,087 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:40:05,088 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:40:06,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:40:07,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:40:09,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:14,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:17,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:40:18,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:40:18,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:40:19,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:20,160 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:40:21,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:40:21,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:40:21,312 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:40:21,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:40:22,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:40:23,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:40:25,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:30,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:32,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:40:33,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:40:33,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:40:35,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:37,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:40:37,490 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:40:37,491 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:40:37,492 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:40:38,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:40:39,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:40:40,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:45,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:47,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:40:48,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:40:48,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:40:50,162 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:40:51,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:40:53,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:40:53,535 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:40:53,536 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:40:53,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:40:54,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:40:55,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:40:56,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:01,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:02,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:41:03,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:41:03,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:41:07,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:09,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:41:09,475 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:41:09,475 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:41:09,476 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:41:10,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:41:11,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:41:13,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:17,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:41:18,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:41:18,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:41:18,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:20,165 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:41:24,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:25,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:41:25,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:41:25,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:41:25,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:41:26,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:41:27,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:41:29,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:32,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:41:33,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:41:33,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:41:35,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:40,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:41,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:41:41,749 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:41:41,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:41:41,750 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:41:42,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:41:43,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:41:45,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:47,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:41:48,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:41:48,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:41:50,168 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:41:51,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:56,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:41:57,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:41:57,815 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:41:57,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:41:57,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:41:58,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:41:59,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:42:01,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:02,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:42:03,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:42:03,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:42:07,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:12,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:13,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:42:13,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:42:13,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:42:13,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:42:14,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:42:15,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:42:17,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:17,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:42:18,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:42:18,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:42:20,170 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:42:23,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:28,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:29,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:42:29,862 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:42:29,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:42:29,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:42:30,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:42:31,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:42:32,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:42:33,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:42:33,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:42:33,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:38,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:43,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:45,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:42:45,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:42:45,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:42:45,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:42:46,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:42:47,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:42:47,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:42:48,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:42:48,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:42:49,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:42:50,174 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:42:55,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:00,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:01,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:43:01,986 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:43:01,987 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:43:01,989 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:43:02,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:43:02,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:43:03,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:03,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:43:03,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:43:05,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:10,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:15,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:17,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:43:17,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:43:17,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:43:17,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:43:17,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:43:18,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:43:18,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:43:18,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:43:19,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:20,179 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:43:21,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:23,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:26,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:27,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:29,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:31,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:32,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:43:33,247 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:33,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:43:33,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:43:35,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:37,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:37,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:43:39,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:42,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:47,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:47,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:43:48,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:43:48,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:43:50,183 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:43:53,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:43:53,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:43:53,672 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:43:53,673 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:43:53,674 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:43:54,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:43:55,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:43:58,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:02,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:44:03,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:44:03,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:44:04,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:09,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:09,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:44:09,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:44:09,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:44:09,915 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:44:10,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:44:11,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:44:14,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:17,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:44:18,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:44:18,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:44:20,185 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:44:20,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:25,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:26,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:44:26,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:44:26,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:44:26,258 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:44:26,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:44:27,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:44:30,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:32,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:44:33,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:44:33,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:44:35,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:40,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:42,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:44:42,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:44:42,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:44:42,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:44:43,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:44:45,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:44:46,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:47,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:44:48,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:44:48,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:44:50,188 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:44:52,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:57,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:44:58,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:44:58,797 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:44:58,798 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:44:58,799 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:44:59,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:45:01,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:45:02,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:02,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:45:03,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:45:03,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:45:08,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:13,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:14,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:45:14,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:45:14,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:45:14,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:45:15,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:45:17,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:45:17,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:45:18,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:45:18,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:45:19,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:20,192 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:45:25,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:30,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:30,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:45:30,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:45:30,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:45:30,992 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:45:31,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:45:32,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:45:33,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:45:33,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:45:33,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:45:35,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:40,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:45,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:47,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:45:47,139 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:45:47,140 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:45:47,142 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:45:47,357 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:45:47,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:45:48,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:45:48,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:45:49,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:45:50,194 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:45:51,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:45:56,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:01,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:02,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:46:03,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:46:03,011 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:46:03,011 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:46:03,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:46:03,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:46:03,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:46:03,370 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:46:05,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:46:06,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:11,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:16,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:17,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:46:18,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:46:18,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:46:18,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:46:18,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:46:18,969 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:46:18,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:46:19,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:46:20,197 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:46:21,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:46:22,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:27,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:32,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:32,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:46:33,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:46:33,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:46:34,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:46:34,910 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:46:34,911 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:46:34,912 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:46:35,395 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:46:37,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:46:37,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:42,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:47,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:46:48,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:46:48,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:46:48,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:50,200 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:46:50,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:46:50,764 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:46:50,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:46:50,765 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:46:51,409 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:46:53,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:46:53,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:46:58,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:02,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:47:03,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:47:03,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:47:04,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:06,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:47:06,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:47:06,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:47:06,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:47:07,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:47:09,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:47:09,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:14,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:17,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:47:18,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:47:18,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:47:20,203 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:47:20,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:22,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:47:22,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:47:22,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:47:22,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:47:23,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:47:25,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:47:25,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:30,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:32,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:47:33,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:47:33,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:47:36,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:38,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:47:38,844 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:47:38,844 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:47:38,845 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:47:39,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:47:41,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:47:41,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:46,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:47,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:47:48,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:47:48,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:47:50,205 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:47:52,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:47:54,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:47:54,807 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:47:54,807 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:47:54,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:47:55,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:47:57,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:47:57,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:02,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:02,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:48:03,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:48:03,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:48:08,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:10,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:48:10,889 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:48:10,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:48:10,891 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:48:11,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:48:13,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:48:13,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:17,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:48:18,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:48:18,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:48:19,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:20,209 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:48:25,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:27,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:48:27,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:48:27,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:48:27,077 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:48:27,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:48:29,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:48:31,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:32,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:48:33,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:48:33,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:48:36,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:41,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:42,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:48:42,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:48:42,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:48:42,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:48:43,498 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:48:45,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:48:46,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:47,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:48:48,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:48:48,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:48:50,213 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:48:52,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:57,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:48:58,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:48:58,968 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:48:58,969 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:48:58,970 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:48:59,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:49:01,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:49:02,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:02,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:49:03,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:49:03,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:49:08,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:13,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:15,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:49:15,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:49:15,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:49:15,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:49:15,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:49:17,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:49:17,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:49:18,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:49:18,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:49:19,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:20,216 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:49:25,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:30,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:31,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:49:31,161 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:49:31,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:49:31,163 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:49:31,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:49:32,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:49:33,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:49:33,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:49:33,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:49:35,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:40,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:45,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:47,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:49:47,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:49:47,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:49:47,227 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:49:47,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:49:47,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:49:48,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:49:48,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:49:49,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:49:50,218 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:49:51,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:49:56,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:01,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:02,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:50:03,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:50:03,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:50:03,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:50:03,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:50:03,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:50:03,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:50:03,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:50:05,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:50:06,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:11,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:16,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:17,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:50:18,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:50:18,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:50:19,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:50:19,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:50:19,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:50:19,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:50:19,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:50:20,222 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:50:21,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:50:22,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:27,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:32,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:32,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:50:33,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:50:33,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:50:35,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:50:35,314 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:50:35,314 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:50:35,315 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:50:35,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:50:37,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:37,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:50:42,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:47,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:47,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:50:48,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:50:48,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:50:50,224 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:50:51,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:50:51,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:50:51,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:50:51,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:50:52,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:50:52,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:50:53,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:50:57,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:02,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:02,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:51:03,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:51:03,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:51:07,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:51:07,591 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:51:07,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:51:07,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:51:07,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:51:08,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:09,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:51:13,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:17,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:51:18,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:51:18,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:51:19,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:20,227 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:51:23,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:51:23,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:51:23,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:51:23,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:51:23,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:51:24,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:25,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:51:29,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:32,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:51:33,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:51:33,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:51:35,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:39,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:51:39,814 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:51:39,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:51:39,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:51:40,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:51:40,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:41,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:51:45,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:47,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:51:48,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:51:48,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:51:50,230 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:51:51,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:55,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:51:55,775 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:51:55,775 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:51:55,777 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:51:56,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:51:56,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:51:57,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:52:01,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:02,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:52:03,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:52:03,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:52:07,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:11,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:52:11,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:52:11,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:52:11,851 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:52:12,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:52:12,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:13,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:52:17,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:17,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:52:18,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:52:18,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:52:20,234 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:52:23,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:27,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:52:27,943 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:52:27,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:52:27,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:52:28,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:52:28,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:29,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:52:32,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:52:33,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:52:33,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:52:34,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:39,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:44,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:52:44,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:52:44,113 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:52:44,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:52:44,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:52:45,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:45,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:52:47,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:52:48,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:52:48,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:52:50,236 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:52:50,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:52:55,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:00,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:00,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:53:00,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:53:00,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:53:00,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:53:00,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:53:01,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:53:02,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:53:03,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:53:03,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:53:05,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:10,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:15,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:16,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:53:16,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:53:16,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:53:16,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:53:16,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:53:17,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:53:17,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:53:18,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:53:18,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:53:20,238 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:53:21,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:26,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:31,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:32,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:53:32,639 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:53:32,639 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:53:32,641 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:53:32,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:53:32,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:53:33,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:53:33,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:53:33,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:53:36,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:41,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:46,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:47,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:53:48,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:53:48,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:53:48,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:53:48,939 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:53:48,940 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:53:48,941 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:53:49,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:53:49,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:53:50,242 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:53:52,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:53:57,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:02,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:02,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:54:03,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:54:03,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:54:05,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:54:05,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:54:05,179 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:54:05,181 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:54:05,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:54:07,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:54:08,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:13,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:17,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:54:18,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:54:18,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:54:18,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:20,244 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:54:21,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:54:21,330 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:54:21,330 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:54:21,332 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:54:21,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:54:23,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:54:24,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:29,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:32,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:54:33,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:54:33,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:54:34,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:37,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:54:37,803 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:54:37,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:54:37,805 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:54:38,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:54:39,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:54:39,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:44,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:47,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:54:48,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:54:48,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:54:50,247 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:54:50,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:54:53,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:54:53,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:54:53,833 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:54:53,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:54:54,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:54:55,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:54:55,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:00,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:02,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:55:03,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:55:03,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:55:06,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:10,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:55:10,058 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:55:10,059 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:55:10,060 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:55:10,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:55:11,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:55:12,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:17,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:17,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:55:18,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:55:18,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:55:20,252 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:55:22,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:26,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:55:26,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:55:26,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:55:26,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:55:26,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:55:27,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:55:28,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:32,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:55:33,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:55:33,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:55:33,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:38,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:42,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:55:42,039 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:55:42,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:55:42,041 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:55:42,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:55:43,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:55:44,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:47,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:55:48,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:55:48,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:55:49,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:50,254 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:55:55,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:55:57,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:55:57,979 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:55:57,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:55:57,981 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:55:58,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:55:59,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:56:00,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:02,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:56:03,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:56:03,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:56:06,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:11,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:13,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:56:13,837 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:56:13,837 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:56:13,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:56:13,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:56:15,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:56:16,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:17,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:56:18,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:56:18,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:56:20,257 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:56:22,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:27,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:29,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:56:29,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:56:29,693 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:56:29,695 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:56:29,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:56:31,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:56:32,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:32,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:56:33,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:56:33,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:56:38,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:43,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:45,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:56:45,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:56:45,733 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:56:45,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:56:45,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:56:47,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:56:47,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:56:48,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:56:48,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:56:49,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:56:50,260 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:56:55,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:00,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:01,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:57:01,594 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:57:01,594 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:57:01,596 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:57:01,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:57:02,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:57:03,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:57:03,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:57:03,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:57:05,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:10,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:15,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:17,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:57:17,574 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:57:17,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:57:17,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:57:17,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:57:17,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:57:18,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:57:18,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:57:19,914 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:57:20,264 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:57:21,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:26,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:31,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:32,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:57:33,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:57:33,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:57:33,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:57:33,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:57:33,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:57:33,617 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:57:33,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:57:35,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:57:36,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:41,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:46,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:47,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:57:48,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:57:48,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:57:49,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:57:49,557 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:57:49,557 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:57:49,559 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:57:49,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:57:50,266 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:57:51,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:57:52,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:57:57,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:02,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:02,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:58:03,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:58:03,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:58:05,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:58:05,654 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:58:05,654 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:58:05,655 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:58:05,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:58:07,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:07,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:58:12,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:17,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:17,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:58:18,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:58:18,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:58:20,269 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:58:21,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:58:21,704 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:58:21,704 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:58:21,705 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:58:21,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:58:22,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:23,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:58:27,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:32,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:32,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:58:33,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:58:33,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:58:37,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:58:37,775 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:58:37,775 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:58:37,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:37,777 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:58:37,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:58:39,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:58:42,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:47,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:47,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:58:48,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:58:48,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:58:50,272 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:58:53,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:58:53,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:58:53,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:58:53,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:58:53,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:58:53,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:58:55,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:58:58,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:02,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:59:03,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:59:03,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:59:04,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:09,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:09,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:59:09,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:59:09,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:59:09,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:59:10,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:59:12,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:59:14,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:17,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:59:18,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:59:18,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:59:20,274 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:59:20,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:25,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:25,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:59:25,739 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:59:25,739 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:59:25,741 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:59:26,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:59:28,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:59:30,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:32,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:59:33,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:59:33,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:59:36,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:41,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:41,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:59:41,912 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:59:41,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:59:41,914 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:59:42,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 04:59:44,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 04:59:46,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:47,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 04:59:48,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 04:59:48,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 04:59:50,277 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 04:59:52,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:57,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 04:59:57,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 04:59:57,875 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 04:59:57,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 04:59:57,877 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 04:59:58,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:00:00,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:00:02,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:02,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:00:03,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:00:03,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:00:08,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:13,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:14,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:00:14,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:00:14,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:00:14,020 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:00:14,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:00:16,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:00:17,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:00:18,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:00:18,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:00:19,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:20,281 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:00:25,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:30,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:00:30,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:00:30,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:00:30,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:00:30,069 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:00:31,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:32,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:00:32,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:00:33,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:00:33,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:00:36,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:41,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:46,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:00:46,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:00:46,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:00:46,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:00:47,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:00:47,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:47,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:00:48,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:00:48,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:00:48,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:00:50,285 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:00:52,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:00:57,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:02,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:02,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:01:02,351 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:01:02,352 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:01:02,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:01:02,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:01:03,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:01:03,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:01:03,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:01:04,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:01:07,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:12,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:17,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:17,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:01:18,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:01:18,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:01:18,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:01:18,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:01:18,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:01:18,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:01:19,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:01:20,111 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:01:20,287 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:01:23,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:28,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:32,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:01:33,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:01:33,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:01:33,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:34,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:01:34,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:01:34,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:01:34,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:01:35,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:01:36,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:01:39,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:44,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:47,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:01:48,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:01:48,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:01:49,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:01:50,290 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:01:50,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:01:50,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:01:50,527 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:01:50,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:01:51,137 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:01:52,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:01:55,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:00,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:02,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:02:03,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:02:03,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:02:06,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:06,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:02:06,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:02:06,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:02:06,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:02:07,151 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:02:08,151 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:02:11,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:16,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:17,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:02:18,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:02:18,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:02:20,294 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:02:22,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:22,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:02:22,537 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:02:22,537 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:02:22,539 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:02:23,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:02:24,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:02:27,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:32,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:32,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:02:33,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:02:33,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:02:38,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:38,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:02:38,719 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:02:38,719 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:02:38,721 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:02:39,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:02:40,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:02:43,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:47,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:02:48,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:02:48,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:02:49,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:50,296 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:02:54,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:02:54,864 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:02:54,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:02:54,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:02:54,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:02:55,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:02:56,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:02:59,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:02,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:03:03,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:03:03,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:03:05,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:10,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:11,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:03:11,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:03:11,003 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:03:11,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:03:11,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:03:12,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:03:16,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:17,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:03:18,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:03:18,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:03:20,300 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:03:21,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:26,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:27,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:03:27,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:03:27,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:03:27,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:03:28,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:03:28,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:03:31,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:32,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:03:33,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:03:33,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:03:36,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:41,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:43,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:03:43,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:03:43,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:03:43,287 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:03:44,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:03:44,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:03:47,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:47,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:03:48,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:03:48,283 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:03:50,302 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:03:52,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:57,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:03:59,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:03:59,530 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:03:59,530 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:03:59,532 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:04:00,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:04:00,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:04:02,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:02,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:04:03,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:04:03,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:04:08,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:13,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:15,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:04:15,747 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:04:15,747 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:04:15,749 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:04:16,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:04:17,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:04:18,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:04:18,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:04:18,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:04:18,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:20,341 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:04:24,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:29,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:31,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:04:31,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:04:31,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:04:31,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:04:32,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:04:32,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:04:33,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:04:33,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:04:34,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:04:34,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:39,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:44,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:47,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:04:48,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:04:48,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:04:48,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:04:48,559 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:04:48,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:04:48,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:04:49,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:04:49,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:04:50,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:04:50,342 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:04:55,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:00,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:02,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:05:03,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:05:03,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:05:04,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:05:04,423 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:05:04,423 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:05:04,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:05:05,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:05:05,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:06,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:05:10,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:15,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:17,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:05:18,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:05:18,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:05:20,346 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:05:20,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:05:20,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:05:20,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:05:20,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:20,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:05:21,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:05:22,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:05:25,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:30,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:32,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:05:33,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:05:33,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:05:36,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:36,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:05:36,601 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:05:36,601 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:05:36,602 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:05:37,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:05:38,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:05:41,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:46,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:47,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:05:48,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:05:48,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:05:50,351 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:05:52,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:05:52,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:05:52,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:05:52,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:05:52,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:05:53,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:05:54,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:05:57,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:02,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:02,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:06:03,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:06:03,284 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:06:08,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:08,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:06:08,795 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:06:08,795 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:06:08,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:06:09,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:06:10,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:06:13,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:17,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:06:18,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:06:18,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:06:19,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:20,352 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:06:24,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:06:24,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:06:24,644 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:06:24,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:24,645 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:06:25,382 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:06:26,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:06:29,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:32,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:06:33,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:06:33,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:06:35,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:40,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:40,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:06:40,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:06:40,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:06:40,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:06:41,396 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:06:42,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:06:45,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:47,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:06:48,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:06:48,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:06:50,355 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:06:51,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:56,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:06:56,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:06:56,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:06:56,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:06:56,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:06:57,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:06:58,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:07:01,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:02,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:07:03,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:07:03,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:07:07,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:12,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:12,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:07:12,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:07:12,627 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:07:12,628 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:07:13,424 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:07:14,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:07:17,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:17,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:07:18,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:07:18,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:07:20,358 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:07:23,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:28,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:28,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:07:28,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:07:28,484 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:07:28,485 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:07:29,437 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:07:30,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:07:32,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:07:33,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:07:33,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:07:33,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:38,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:43,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:44,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:07:44,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:07:44,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:07:44,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:07:45,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:07:46,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:07:47,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:07:48,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:07:48,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:07:48,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:50,362 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:07:54,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:07:59,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:00,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:08:00,504 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:08:00,505 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:08:00,506 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:08:01,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:08:02,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:08:02,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:08:03,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:08:03,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:08:04,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:09,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:14,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:16,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:08:16,557 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:08:16,558 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:08:16,559 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:08:17,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:08:17,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:08:18,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:08:18,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:08:18,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:08:19,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:20,365 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:08:25,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:30,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:32,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:08:32,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:08:32,631 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:08:32,632 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:08:32,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:08:33,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:08:33,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:08:33,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:08:34,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:08:35,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:40,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:45,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:47,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:08:48,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:08:48,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:08:48,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:08:48,588 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:08:48,588 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:08:48,590 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:08:49,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:08:50,368 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:08:50,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:08:51,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:08:56,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:01,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:02,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:09:03,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:09:03,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:09:04,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:09:04,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:09:04,605 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:09:04,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:09:05,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:09:06,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:09:06,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:11,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:16,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:17,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:09:18,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:09:18,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:09:20,373 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:09:20,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:09:20,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:09:20,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:09:20,766 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:09:21,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:09:21,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:22,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:09:26,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:31,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:32,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:09:33,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:09:33,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:09:36,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:09:36,815 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:09:36,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:09:36,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:36,817 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:09:37,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:09:38,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:09:41,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:46,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:47,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:09:48,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:09:48,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:09:50,374 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:09:52,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:09:52,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:09:52,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:09:52,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:09:52,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:09:53,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:09:54,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:09:57,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:02,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:02,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:10:03,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:10:03,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:10:08,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:08,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:10:08,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:10:08,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:10:08,858 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:10:09,576 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:10:10,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:10:13,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:17,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:10:18,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:10:18,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:10:19,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:20,377 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:10:25,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:10:25,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:10:25,017 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:10:25,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:25,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:10:25,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:10:26,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:10:30,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:32,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:10:33,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:10:33,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:10:35,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:40,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:41,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:10:41,484 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:10:41,484 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:10:41,486 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:10:41,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:10:42,604 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:10:45,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:47,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:10:48,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:10:48,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:10:50,379 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:10:51,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:56,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:10:57,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:10:57,334 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:10:57,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:10:57,335 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:10:57,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:10:58,618 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:11:02,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:02,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:11:03,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:11:03,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:11:07,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:12,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:13,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:11:13,577 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:11:13,577 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:11:13,579 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:11:13,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:11:14,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:11:17,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:17,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:11:18,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:11:18,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:11:20,383 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:11:23,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:28,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:29,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:11:29,622 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:11:29,623 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:11:29,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:11:29,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:11:30,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:11:32,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:11:33,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:11:33,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:11:33,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:38,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:43,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:45,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:11:45,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:11:45,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:11:45,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:11:45,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:11:46,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:11:47,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:11:48,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:11:48,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:11:48,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:50,386 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:11:54,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:11:59,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:02,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:12:02,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:12:02,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:12:02,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:12:02,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:12:02,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:12:02,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:12:03,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:12:03,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:12:04,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:09,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:14,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:17,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:12:18,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:12:18,089 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:12:18,089 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:12:18,090 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:12:18,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:12:18,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:12:18,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:12:18,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:12:20,389 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:12:20,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:25,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:30,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:32,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:12:33,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:12:33,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:12:34,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:12:34,040 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:12:34,040 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:12:34,042 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:12:34,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:12:34,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:12:36,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:41,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:46,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:47,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:12:48,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:12:48,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:12:50,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:12:50,196 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:12:50,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:12:50,197 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:12:50,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:12:50,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:12:50,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:12:51,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:12:56,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:01,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:02,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:13:03,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:13:03,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:13:06,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:13:06,342 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:13:06,342 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:13:06,344 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:13:06,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:13:07,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:08,732 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:13:12,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:17,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:17,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:13:18,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:13:18,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:13:20,395 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:13:22,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:22,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:13:22,668 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:13:22,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:13:22,670 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:13:22,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:13:24,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:13:27,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:32,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:32,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:13:33,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:13:33,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:13:38,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:38,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:13:38,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:13:38,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:13:38,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:13:39,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:13:40,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:13:43,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:47,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:13:48,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:13:48,287 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:13:49,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:50,398 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:13:54,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:13:54,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:13:54,829 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:13:54,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:13:54,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:13:55,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:13:56,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:13:59,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:02,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:14:03,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:14:03,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:14:05,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:10,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:10,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:14:10,959 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:14:10,959 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:14:10,961 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:14:11,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:14:12,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:14:15,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:17,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:14:18,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:14:18,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:14:20,403 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:14:21,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:26,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:26,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:14:27,000 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:14:27,001 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:14:27,002 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:14:27,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:14:28,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:14:32,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:32,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:14:33,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:14:33,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:14:37,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:42,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:43,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:14:43,246 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:14:43,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:14:43,248 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:14:43,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:14:44,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:14:47,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:47,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:14:48,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:14:48,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:14:50,405 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:14:53,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:58,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:14:59,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:14:59,488 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:14:59,488 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:14:59,490 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:14:59,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:15:00,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:15:02,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:15:03,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:15:03,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:15:03,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:08,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:13,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:15,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:15:15,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:15:15,542 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:15:15,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:15:15,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:15:16,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:15:17,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:15:18,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:15:18,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:15:19,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:20,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:15:25,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:30,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:31,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:15:31,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:15:31,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:15:31,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:15:31,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:15:32,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:15:32,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:15:33,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:15:33,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:15:35,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:40,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:45,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:47,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:15:47,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:15:47,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:15:47,828 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:15:47,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:15:47,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:15:48,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:15:48,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:15:48,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:15:50,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:15:51,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:15:56,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:01,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:02,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:16:03,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:16:03,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:16:03,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:16:03,848 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:16:03,848 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:16:03,850 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:16:03,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:16:04,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:16:06,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:11,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:16,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:17,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:16:18,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:16:18,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:16:19,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:16:19,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:16:19,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:16:19,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:16:20,415 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:16:20,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:16:20,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:16:22,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:27,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:32,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:32,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:16:33,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:16:33,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:16:35,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:16:35,783 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:16:35,783 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:16:35,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:16:35,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:16:36,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:16:37,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:42,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:47,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:47,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:16:48,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:16:48,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:16:50,417 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:16:51,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:16:51,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:16:51,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:16:51,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:16:51,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:16:52,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:16:52,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:16:57,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:02,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:02,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:17:03,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:17:03,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:17:07,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:17:07,862 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:17:07,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:17:07,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:07,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:17:07,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:17:08,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:17:12,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:17,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:17,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:17:18,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:17:18,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:17:20,420 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:17:23,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:23,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:17:23,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:17:23,819 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:17:23,820 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:17:23,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:17:24,970 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:17:28,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:32,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:17:33,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:17:33,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:17:34,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:39,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:39,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:17:39,766 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:17:39,767 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:17:39,768 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:17:39,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:17:40,984 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:17:44,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:47,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:17:48,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:17:48,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:17:50,424 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:17:50,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:55,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:17:55,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:17:55,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:17:55,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:17:55,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:17:55,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:17:56,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:18:00,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:02,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:18:03,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:18:03,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:18:06,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:11,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:11,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:18:11,904 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:18:11,904 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:18:11,906 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:18:12,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:18:13,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:18:16,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:17,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:18:18,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:18:18,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:18:20,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:18:22,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:27,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:27,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:18:27,886 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:18:27,886 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:18:27,887 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:18:28,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:18:29,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:18:32,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:32,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:18:33,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:18:33,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:18:38,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:43,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:44,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:18:44,069 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:18:44,070 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:18:44,071 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:18:45,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:18:45,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:18:47,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:18:48,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:18:48,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:18:49,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:18:50,430 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:18:55,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:00,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:19:00,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:19:00,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:19:00,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:19:00,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:19:01,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:01,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:19:02,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:19:03,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:19:03,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:19:06,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:11,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:15,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:19:15,985 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:19:15,986 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:19:15,988 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:19:16,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:19:16,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:17,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:19:17,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:19:18,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:19:18,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:19:20,434 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:19:22,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:27,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:32,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:19:32,035 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:19:32,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:19:32,079 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:19:32,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:19:32,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:32,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:19:33,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:19:33,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:19:33,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:19:38,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:43,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:47,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:19:48,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:19:48,190 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:19:48,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:19:48,192 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:19:48,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:19:48,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:19:49,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:19:49,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:19:49,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:19:50,436 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:19:55,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:00,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:02,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:20:03,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:20:03,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:20:04,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:20:04,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:20:04,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:20:04,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:20:05,109 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:20:05,109 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:20:06,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:11,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:16,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:17,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:20:18,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:20:18,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:20:20,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:20:20,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:20:20,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:20:20,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:20:20,439 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:20:21,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:20:21,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:20:21,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:26,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:31,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:32,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:20:33,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:20:33,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:20:36,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:20:36,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:20:36,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:20:36,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:36,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:20:37,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:20:37,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:20:41,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:46,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:47,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:20:48,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:20:48,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:20:50,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:20:52,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:20:52,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:20:52,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:20:52,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:20:52,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:20:53,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:20:53,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:20:57,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:02,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:02,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:21:03,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:21:03,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:21:08,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:21:08,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:21:08,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:21:08,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:08,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:21:09,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:21:09,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:21:13,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:17,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:21:18,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:21:18,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:21:18,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:20,447 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:21:24,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:24,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:21:24,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:21:24,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:21:24,786 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:21:25,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:21:27,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:21:29,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:32,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:21:33,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:21:33,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:21:35,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:40,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:40,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:21:40,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:21:40,820 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:21:40,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:21:41,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:21:43,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:21:45,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:47,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:21:48,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:21:48,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:21:50,447 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:21:51,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:56,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:21:56,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:21:56,794 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:21:56,795 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:21:56,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:21:57,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:21:57,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:22:01,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:02,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:22:03,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:22:03,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:22:07,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:12,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:13,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:22:13,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:22:13,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:22:13,034 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:22:13,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:22:15,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:22:17,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:17,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:22:18,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:22:18,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:22:20,450 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:22:23,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:28,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:28,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:22:28,977 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:22:28,978 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:22:28,980 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:22:29,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:22:31,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:22:32,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:22:33,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:22:33,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:22:33,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:38,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:43,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:44,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:22:44,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:22:44,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:22:44,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:22:45,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:22:47,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:22:47,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:22:48,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:22:48,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:22:49,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:22:50,455 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:22:55,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:00,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:01,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:23:01,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:23:01,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:23:01,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:23:01,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:23:02,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:23:03,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:23:03,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:23:03,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:23:05,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:10,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:15,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:17,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:23:17,174 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:23:17,174 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:23:17,176 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:23:17,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:23:17,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:23:18,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:23:18,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:23:19,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:23:20,457 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:23:21,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:26,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:31,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:32,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:23:33,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:23:33,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:23:33,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:23:33,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:23:33,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:23:33,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:23:34,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:23:35,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:23:36,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:41,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:46,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:47,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:23:48,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:23:48,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:23:49,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:23:49,346 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:23:49,346 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:23:49,348 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:23:50,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:23:50,459 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:23:51,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:23:52,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:23:57,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:02,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:02,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:24:03,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:24:03,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:24:05,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:24:05,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:24:05,423 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:24:05,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:24:06,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:24:07,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:24:08,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:13,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:17,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:24:18,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:24:18,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:24:18,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:20,462 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:24:21,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:24:21,761 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:24:21,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:24:21,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:24:22,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:24:23,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:24:23,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:28,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:32,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:24:33,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:24:33,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:24:34,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:37,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:24:37,793 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:24:37,793 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:24:37,795 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:24:38,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:24:39,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:24:39,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:44,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:47,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:24:48,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:24:48,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:24:50,466 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:24:50,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:24:54,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:24:54,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:24:54,072 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:24:54,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:24:54,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:24:55,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:24:56,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:01,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:02,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:25:03,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:25:03,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:25:06,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:10,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:25:10,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:25:10,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:25:10,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:25:10,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:25:11,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:25:12,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:17,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:17,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:25:18,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:25:18,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:25:20,469 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:25:22,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:26,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:25:26,314 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:25:26,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:25:26,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:25:26,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:25:27,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:25:28,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:32,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:25:33,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:25:33,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:25:33,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:38,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:42,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:25:42,548 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:25:42,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:25:42,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:25:43,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:25:43,393 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:25:44,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:47,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:25:48,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:25:48,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:25:50,472 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:25:50,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:55,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:25:58,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:25:58,576 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:25:58,577 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:25:58,578 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:25:59,408 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:25:59,408 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:26:00,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:02,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:26:03,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:26:03,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:26:06,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:11,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:14,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:26:14,602 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:26:14,602 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:26:14,604 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:26:15,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:26:15,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:26:16,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:17,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:26:18,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:26:18,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:26:20,476 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:26:22,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:27,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:30,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:26:30,782 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:26:30,782 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:26:30,784 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:26:31,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:26:32,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:26:32,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:32,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:26:33,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:26:33,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:26:38,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:43,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:46,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:26:46,636 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:26:46,636 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:26:46,639 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:26:47,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:26:47,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:26:48,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:26:48,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:26:48,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:26:48,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:50,485 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:26:54,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:26:59,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:02,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:27:02,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:27:02,707 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:27:02,708 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:27:02,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:27:03,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:27:03,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:27:03,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:27:04,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:27:04,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:09,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:14,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:17,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:27:18,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:27:18,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:27:18,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:27:18,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:27:18,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:27:18,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:27:19,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:27:19,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:20,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:27:20,487 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:27:25,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:30,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:32,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:27:33,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:27:33,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:27:34,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:27:34,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:27:34,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:27:34,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:27:35,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:27:35,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:36,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:27:40,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:45,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:47,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:27:48,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:27:48,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:27:50,491 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:27:50,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:27:50,769 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:27:50,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:27:50,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:27:51,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:27:51,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:27:52,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:27:56,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:01,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:02,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:28:03,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:28:03,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:28:07,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:28:07,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:28:07,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:28:07,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:07,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:28:07,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:28:08,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:28:12,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:17,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:17,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:28:18,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:28:18,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:28:20,494 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:28:22,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:23,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:28:23,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:28:23,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:28:23,168 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:28:23,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:28:24,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:28:28,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:32,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:28:33,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:28:33,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:28:33,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:38,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:39,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:28:39,095 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:28:39,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:28:39,097 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:28:39,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:28:40,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:28:44,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:47,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:28:48,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:28:48,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:28:49,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:50,497 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:28:54,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:28:55,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:28:55,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:28:55,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:28:55,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:28:55,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:28:58,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:29:00,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:02,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:29:03,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:29:03,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:29:05,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:10,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:11,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:29:11,398 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:29:11,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:29:11,399 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:29:11,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:29:14,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:29:16,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:17,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:29:18,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:29:18,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:29:20,500 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:29:21,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:26,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:27,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:29:27,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:29:27,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:29:27,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:29:27,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:29:30,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:29:32,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:32,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:29:33,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:29:33,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:29:37,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:42,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:43,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:29:43,472 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:29:43,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:29:43,473 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:29:43,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:29:46,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:29:47,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:47,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:29:48,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:29:48,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:29:50,504 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:29:53,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:58,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:29:59,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:29:59,687 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:29:59,687 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:29:59,689 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:30:00,603 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:30:02,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:30:02,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:30:03,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:30:03,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:30:03,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:08,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:13,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:15,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:30:15,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:30:15,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:30:15,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:30:16,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:30:17,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:30:18,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:30:18,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:30:18,618 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:30:19,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:20,506 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:30:25,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:30,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:31,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:30:31,792 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:30:31,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:30:31,794 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:30:32,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:30:32,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:30:33,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:30:33,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:30:34,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:30:35,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:40,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:45,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:47,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:30:48,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:30:48,009 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:30:48,009 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:30:48,011 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:30:48,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:30:48,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:30:48,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:30:50,509 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:30:50,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:30:51,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:30:56,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:01,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:02,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:31:03,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:31:03,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:31:04,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:31:04,147 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:31:04,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:31:04,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:31:04,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:31:06,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:31:07,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:12,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:17,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:17,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:31:18,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:31:18,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:31:20,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:31:20,116 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:31:20,116 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:31:20,118 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:31:20,514 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:31:20,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:31:22,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:22,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:31:27,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:32,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:32,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:31:33,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:31:33,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:31:36,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:31:36,407 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:31:36,407 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:31:36,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:31:36,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:31:38,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:38,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:31:43,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:47,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:31:48,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:31:48,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:31:48,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:50,516 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:31:52,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:31:52,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:31:52,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:31:52,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:31:52,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:31:54,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:31:54,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:31:59,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:02,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:32:03,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:32:03,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:32:04,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:08,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:32:08,290 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:32:08,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:32:08,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:32:08,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:32:10,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:10,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:32:15,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:17,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:32:18,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:32:18,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:32:20,519 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:32:20,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:24,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:32:24,525 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:32:24,526 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:32:24,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:32:24,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:32:25,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:26,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:32:30,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:32,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:32:33,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:32:33,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:32:35,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:40,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:32:40,477 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:32:40,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:32:40,479 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:32:40,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:32:41,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:42,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:32:46,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:47,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:32:48,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:32:48,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:32:50,522 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:32:51,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:56,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:32:56,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:32:56,638 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:32:56,639 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:32:56,640 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:32:56,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:32:58,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:33:01,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:02,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:33:03,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:33:03,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:33:07,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:12,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:12,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:33:12,707 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:33:12,708 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:33:12,709 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:33:12,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:33:14,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:33:17,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:17,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:33:18,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:33:18,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:33:23,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 05:33:23,579 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:33:23,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:28,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:28,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:33:28,773 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:33:28,774 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:33:28,775 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:33:29,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:33:30,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:33:32,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:33:33,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:33:33,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:33:34,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:39,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:44,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:44,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:33:44,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:33:44,921 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:33:44,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:33:45,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:33:46,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:33:47,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:33:48,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:33:48,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:33:50,527 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:33:50,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:33:55,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:00,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:01,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:34:01,056 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:34:01,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:34:01,058 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:34:01,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:34:02,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:34:02,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:34:03,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:34:03,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:34:06,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:11,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:16,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:17,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:34:17,128 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:34:17,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:34:17,130 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:34:17,806 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:34:17,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:34:18,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:34:18,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:34:18,807 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:34:20,531 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:34:21,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:26,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:31,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:32,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:34:33,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:34:33,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:34:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:34:33,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:34:33,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:34:33,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:34:33,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:34:34,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:34:36,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:41,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:46,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:47,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:34:48,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:34:48,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:34:49,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:34:49,375 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:34:49,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:34:49,377 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:34:49,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:34:50,535 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:34:50,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:34:52,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:34:57,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:02,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:02,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:35:03,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:35:03,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:35:05,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:35:05,611 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:35:05,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:35:05,614 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:35:05,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:35:06,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:35:07,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:12,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:17,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:17,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:35:18,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:35:18,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:35:20,539 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:35:21,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:35:21,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:35:21,969 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:35:21,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:35:22,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:35:22,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:24,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:35:27,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:32,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:35:33,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:35:33,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:35:33,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:38,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:35:38,118 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:35:38,118 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:35:38,121 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:35:38,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:35:39,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:40,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:35:44,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:47,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:35:48,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:35:48,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:35:49,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:50,542 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:35:54,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:35:54,263 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:35:54,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:35:54,265 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:35:54,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:35:55,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:35:56,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:36:00,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:02,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:36:03,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:36:03,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:36:05,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:10,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:36:10,329 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:36:10,329 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:36:10,331 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:36:10,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:36:11,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:12,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:36:16,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:17,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:36:18,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:36:18,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:36:20,545 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:36:21,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:26,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:36:26,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:36:26,471 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:36:26,472 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:36:26,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:36:27,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:28,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:36:32,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:32,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:36:33,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:36:33,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:36:37,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:42,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:36:42,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:36:42,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:36:42,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:36:42,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:36:43,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:44,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:36:47,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:36:48,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:36:48,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:36:48,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:50,548 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:36:54,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:36:58,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:36:58,538 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:36:58,539 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:36:58,541 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:36:58,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:37:00,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:00,938 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:02,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:37:03,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:37:03,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:37:06,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:11,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:14,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:37:14,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:37:14,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:37:14,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:37:14,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:37:16,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:17,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:18,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:37:18,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:37:18,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:37:18,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:20,551 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:37:22,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:23,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:24,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:25,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:28,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:30,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:32,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:37:33,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:37:33,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:37:33,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:34,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:37:36,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:39,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:44,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:48,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:37:48,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:37:48,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:37:49,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:50,554 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:37:50,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:37:50,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:37:50,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:37:50,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:37:50,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:37:52,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:37:54,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:37:59,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:03,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:38:03,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:38:03,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:38:05,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:06,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:38:06,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:38:06,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:38:06,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:38:06,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:38:08,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:38:10,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:15,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:38:18,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:38:18,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:38:20,558 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:38:21,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:23,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:38:23,133 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:38:23,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:38:23,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:38:24,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:38:25,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:38:27,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:32,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:38:33,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:38:33,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:38:37,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:39,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:38:39,170 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:38:39,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:38:39,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:38:40,016 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:38:41,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:38:43,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:48,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:48,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:38:48,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:38:48,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:38:50,588 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:38:53,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:38:55,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:38:55,457 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:38:55,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:38:55,459 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:38:56,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:38:57,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:38:59,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:03,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:39:03,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:39:03,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:39:04,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:09,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:11,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:39:11,457 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:39:11,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:39:11,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:39:12,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:39:13,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:39:15,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:39:18,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:39:18,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:39:20,591 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:39:20,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:25,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:27,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:39:27,407 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:39:27,408 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:39:27,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:39:28,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:39:29,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:39:31,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:39:33,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:39:33,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:39:36,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:41,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:43,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:39:43,630 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:39:43,631 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:39:43,632 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:39:44,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:39:45,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:39:46,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:48,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:39:48,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:39:48,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:39:50,596 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:39:52,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:57,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:39:59,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:39:59,590 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:39:59,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:39:59,592 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:40:00,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:40:01,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:40:03,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:03,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:40:03,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:40:03,334 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:40:08,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:13,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:15,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:40:15,442 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:40:15,442 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:40:15,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:40:16,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:40:17,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:40:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:40:18,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:40:18,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:40:19,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:20,597 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:40:24,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:29,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:31,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:40:31,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:40:31,544 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:40:31,545 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:40:32,109 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:40:33,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:40:33,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:40:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:40:33,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:40:34,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:39,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:44,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:47,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:40:47,518 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:40:47,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:40:47,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:40:48,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:40:48,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:40:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:40:48,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:40:49,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:40:50,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:40:50,601 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:40:55,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:00,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:03,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:41:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:41:03,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:41:03,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:41:03,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:41:03,585 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:41:03,586 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:41:04,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:41:05,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:41:06,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:11,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:16,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:18,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:41:18,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:41:18,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:41:19,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:41:19,449 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:41:19,449 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:41:19,451 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:41:20,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:41:20,604 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:41:21,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:41:21,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:26,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:31,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:41:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:41:33,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:41:35,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:41:35,514 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:41:35,514 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:41:35,516 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:41:36,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:41:37,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:41:37,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:42,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:47,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:48,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:41:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:41:48,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:41:50,607 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:41:51,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:41:51,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:41:51,665 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:41:51,666 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:41:52,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:41:52,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:41:53,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:41:57,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:02,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:03,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:42:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:42:03,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:42:07,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:42:07,714 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:42:07,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:42:07,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:07,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:42:08,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:42:09,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:42:12,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:17,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:18,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:42:18,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:42:18,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:42:20,610 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:42:23,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:23,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:42:23,690 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:42:23,690 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:42:23,692 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:42:24,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:42:25,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:42:28,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:42:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:42:33,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:42:34,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:39,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:39,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:42:39,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:42:39,833 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:42:39,835 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:42:40,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:42:41,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:42:44,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:48,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:42:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:42:48,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:42:50,613 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:42:50,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:55,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:42:55,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:42:55,799 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:42:55,799 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:42:55,800 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:42:56,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:42:57,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:43:00,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:03,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:43:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:43:03,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:43:06,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:11,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:11,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:43:11,757 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:43:11,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:43:11,759 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:43:12,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:43:13,239 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:43:16,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:18,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:43:18,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:43:18,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:43:20,617 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:43:22,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:27,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:27,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:43:28,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:43:28,002 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:43:28,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:43:28,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:43:29,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:43:33,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:43:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:43:33,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:43:38,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:43,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:44,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:43:44,059 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:43:44,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:43:44,061 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:43:44,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:43:45,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:43:48,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:43:48,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:43:48,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:43:48,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:50,619 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:43:53,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:43:58,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:00,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:44:00,099 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:44:00,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:44:00,101 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:44:00,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:44:01,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:44:03,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:44:03,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:44:03,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:44:04,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:09,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:14,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:16,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:44:16,060 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:44:16,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:44:16,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:44:16,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:44:17,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:44:18,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:44:18,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:44:18,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:44:19,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:20,621 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:44:24,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:29,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:32,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:44:32,199 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:44:32,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:44:32,201 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:44:32,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:44:33,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:44:33,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:44:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:44:33,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:44:34,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:39,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:44,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:48,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:44:48,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:44:48,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:44:48,355 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:44:48,356 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:44:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:44:48,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:44:49,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:44:50,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:44:50,624 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:44:51,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:44:55,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:00,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:03,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:45:03,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:45:03,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:45:04,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:45:04,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:45:04,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:45:04,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:45:05,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:45:06,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:07,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:45:11,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:16,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:18,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:45:18,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:45:18,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:45:20,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:45:20,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:45:20,319 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:45:20,320 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:45:20,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:45:20,627 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:45:21,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:45:21,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:26,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:31,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:33,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:45:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:45:33,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:45:36,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:45:36,867 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:45:36,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:45:36,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:36,869 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:45:37,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:45:39,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:45:41,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:46,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:48,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:45:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:45:48,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:45:50,630 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:45:52,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:45:52,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:45:52,716 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:45:52,716 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:45:52,718 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:45:53,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:45:55,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:45:57,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:02,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:03,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:46:03,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:46:03,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:46:08,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:08,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:46:08,885 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:46:08,886 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:46:08,887 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:46:09,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:46:11,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:46:13,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:18,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:46:18,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:46:18,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:46:19,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:20,634 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:46:24,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:25,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:46:25,156 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:46:25,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:46:25,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:46:25,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:46:27,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:46:30,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:33,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:46:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:46:33,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:46:35,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:40,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:41,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:46:41,206 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:46:41,207 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:46:41,208 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:46:41,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:46:43,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:46:46,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:48,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:46:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:46:48,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:46:50,638 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:46:51,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:56,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:46:57,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:46:57,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:46:57,474 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:46:57,476 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:46:58,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:46:59,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:47:02,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:03,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:47:03,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:47:03,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:47:07,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:12,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:13,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:47:13,648 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:47:13,648 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:47:13,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:47:14,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:47:15,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:47:17,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:18,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:47:18,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:47:18,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:47:20,639 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:47:23,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:28,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:30,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:47:30,055 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:47:30,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:47:30,057 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:47:30,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:47:31,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:47:33,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:47:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:47:33,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:47:34,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:39,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:44,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:45,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:47:45,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:47:45,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:47:45,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:47:46,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:47:47,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:47:48,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:47:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:47:48,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:47:49,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:50,642 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:47:54,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:47:59,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:02,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:48:02,368 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:48:02,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:48:02,370 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:48:02,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:48:03,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:48:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:48:03,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:48:03,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:48:05,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:10,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:15,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:18,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:48:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:48:18,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:48:18,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:48:18,654 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:48:18,654 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:48:18,656 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:48:19,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:48:20,647 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:48:20,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:21,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:48:25,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:30,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:48:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:48:33,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:48:34,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:48:34,814 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:48:34,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:48:34,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:48:35,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:48:35,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:37,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:48:40,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:45,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:48:48,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:48:48,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:48:50,649 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:48:50,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:48:50,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:48:50,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:48:50,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:48:50,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:48:51,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:48:53,521 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:48:55,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:00,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:49:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:49:03,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:49:06,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:07,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:49:07,259 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:49:07,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:49:07,261 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:49:07,532 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:49:09,533 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:49:12,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:17,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:18,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:49:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:49:18,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:49:20,652 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:49:22,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:23,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:49:23,368 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:49:23,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:49:23,370 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:49:23,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:49:25,545 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:49:28,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:49:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:49:33,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:49:33,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:38,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:39,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:49:39,299 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:49:39,300 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:49:39,301 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:49:39,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:49:41,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:49:44,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:49:48,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:49:48,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:49:49,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:50,654 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:49:54,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:49:55,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:49:55,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:49:55,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:49:55,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:49:55,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:49:57,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:50:00,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:50:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:50:03,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:50:05,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:10,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:11,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:50:11,477 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:50:11,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:50:11,479 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:50:11,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:50:13,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:50:16,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:18,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:50:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:50:18,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:50:20,659 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:50:21,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:26,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:27,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:50:27,365 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:50:27,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:50:27,367 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:50:27,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:50:29,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:50:32,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:33,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:50:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:50:33,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:50:37,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:42,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:43,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:50:43,412 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:50:43,413 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:50:43,415 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:50:43,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:50:45,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:50:48,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:48,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:50:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:50:48,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:50:50,664 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:50:53,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:58,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:50:59,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:50:59,267 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:50:59,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:50:59,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:50:59,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:51:01,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:51:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:51:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:51:03,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:51:04,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:09,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:14,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:15,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:51:15,321 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:51:15,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:51:15,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:51:15,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:51:17,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:51:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:51:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:51:18,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:51:20,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:20,663 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:51:25,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:30,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:31,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:51:31,273 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:51:31,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:51:31,275 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:51:31,646 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:51:33,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:51:33,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:51:33,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:51:33,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:51:36,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:41,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:46,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:47,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:51:47,344 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:51:47,345 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:51:47,346 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:51:47,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:51:48,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:51:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:51:48,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:51:49,661 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:51:50,668 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:51:51,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:51:56,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:01,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:03,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:52:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:52:03,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:52:03,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:52:03,595 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:52:03,595 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:52:03,597 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:52:03,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:52:05,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:52:07,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:12,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:17,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:52:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:52:18,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:52:19,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:52:19,656 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:52:19,656 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:52:19,658 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:52:19,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:52:20,670 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:52:21,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:52:22,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:27,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:32,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:33,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:52:33,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:52:33,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:52:35,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:52:35,498 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:52:35,499 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:52:35,501 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:52:35,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:52:37,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:52:38,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:43,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:48,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:52:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:52:48,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:52:48,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:50,673 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:52:51,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:52:51,652 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:52:51,652 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:52:51,654 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:52:51,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:52:53,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:52:53,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:52:58,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:03,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:53:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:53:03,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:53:04,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:07,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:53:07,766 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:53:07,766 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:53:07,768 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:53:08,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:53:09,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:53:09,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:14,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:18,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:53:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:53:18,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:53:20,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:20,676 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:53:23,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:53:23,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:53:23,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:53:23,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:53:24,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:53:25,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:53:25,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:30,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:53:33,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:53:33,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:53:36,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:39,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:53:39,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:53:39,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:53:39,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:53:40,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:53:41,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:53:41,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:46,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:53:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:53:48,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:53:50,679 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:53:52,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:53:55,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:53:55,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:53:55,950 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:53:55,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:53:56,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:53:57,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:53:57,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:02,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:03,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:54:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:54:03,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:54:08,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:12,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:54:12,172 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:54:12,172 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:54:12,174 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:54:12,776 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:54:13,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:54:14,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:18,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:54:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:54:18,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:54:19,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:20,682 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:54:24,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:28,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:54:28,024 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:54:28,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:54:28,026 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:54:28,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:54:29,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:54:30,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:54:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:54:33,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:54:35,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:40,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:44,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:54:44,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:54:44,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:54:44,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:54:44,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:54:45,803 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:54:46,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:54:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:54:48,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:54:50,685 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:54:51,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:54:56,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:00,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:55:00,437 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:55:00,438 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:55:00,439 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:55:00,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:55:01,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:55:02,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:03,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:55:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:55:03,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:55:07,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:12,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:16,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:55:16,379 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:55:16,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:55:16,381 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:55:16,829 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:55:17,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:55:18,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:55:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:55:18,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:55:20,689 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:55:23,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:28,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:32,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:55:32,515 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:55:32,515 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:55:32,517 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:55:32,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:55:33,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:55:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:55:33,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:55:33,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:55:34,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:39,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:44,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:48,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:55:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:55:48,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:55:48,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:55:48,898 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:55:48,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:55:48,900 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:55:49,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:55:49,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:55:49,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:55:50,690 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:55:55,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:00,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:03,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:56:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:56:03,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:56:05,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:56:05,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:56:05,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:56:05,026 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:56:05,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:56:05,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:56:06,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:11,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:16,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:56:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:56:18,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:56:20,693 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:56:20,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:56:20,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:56:20,956 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:56:20,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:56:21,888 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:56:21,888 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:56:21,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:26,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:31,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:33,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:56:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:56:33,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:56:37,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:56:37,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:56:37,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:56:37,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:37,441 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:56:37,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:56:39,902 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:56:42,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:47,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:48,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:56:48,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:56:48,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:56:50,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:56:52,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:56:53,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:56:53,391 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:56:53,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:56:53,393 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:56:53,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:56:55,914 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:56:58,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:03,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:57:03,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:57:03,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:57:03,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:08,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:09,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:57:09,590 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:57:09,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:57:09,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:57:09,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:57:11,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:57:13,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:57:18,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:57:18,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:57:18,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:20,701 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:57:23,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:25,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:57:25,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:57:25,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:57:25,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:57:25,938 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:57:27,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:57:28,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:33,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:57:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:57:33,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:57:34,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:39,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:42,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:57:42,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:57:42,013 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:57:42,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:57:42,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:57:43,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:57:45,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:48,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:57:48,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:57:48,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:57:50,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:50,704 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:57:55,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:57:58,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:57:58,209 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:57:58,209 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:57:58,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:57:58,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:57:59,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:58:01,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:03,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:58:03,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:58:03,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:58:06,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:11,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:14,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:58:14,345 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:58:14,345 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:58:14,346 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:58:14,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:58:15,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:58:17,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:18,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:58:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:58:18,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:58:20,707 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:58:22,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:27,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:30,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:58:30,792 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:58:30,793 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:58:30,794 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:58:30,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:58:31,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:58:32,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:33,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:58:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:58:33,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:58:38,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:43,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:46,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:58:46,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:58:46,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:58:46,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:58:47,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:58:48,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:58:48,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:58:48,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:58:48,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:58:48,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:50,713 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:58:53,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:58:58,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:02,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:59:02,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:59:02,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:59:02,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:59:03,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:59:03,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:59:03,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:59:03,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:59:04,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:59:04,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:09,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:14,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:18,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:59:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:59:18,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:59:18,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:59:19,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:59:19,001 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:59:19,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:59:19,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:59:20,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:20,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:59:20,715 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:59:25,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:30,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:33,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:59:33,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:59:33,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:59:34,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:59:34,962 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:59:34,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:59:34,964 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:59:35,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:59:35,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:36,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:59:40,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:45,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:48,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 05:59:48,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 05:59:48,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 05:59:50,718 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 05:59:50,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 05:59:51,000 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 05:59:51,000 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 05:59:51,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 05:59:51,002 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 05:59:51,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 05:59:52,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 05:59:56,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:01,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:03,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:00:03,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:00:03,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:00:06,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:07,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:00:07,129 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:00:07,130 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:00:07,131 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:00:08,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:00:08,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:00:12,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:17,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:18,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:00:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:00:18,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:00:20,723 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:00:22,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:23,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:00:23,092 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:00:23,093 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:00:23,094 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:00:24,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:00:24,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:00:28,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:33,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:33,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:00:33,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:00:33,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:00:38,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:39,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:00:39,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:00:39,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:00:39,253 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:00:40,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:00:40,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:00:44,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:48,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:00:48,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:00:48,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:00:49,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:50,725 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:00:54,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:00:55,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:00:55,312 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:00:55,312 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:00:55,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:00:56,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:00:56,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:01:00,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:03,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:01:03,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:01:03,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:01:05,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:10,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:11,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:01:11,161 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:01:11,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:01:11,163 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:01:12,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:01:12,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:01:16,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:18,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:01:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:01:18,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:01:20,727 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:01:21,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:26,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:27,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:01:27,208 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:01:27,209 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:01:27,210 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:01:28,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:01:28,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:01:32,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:33,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:01:33,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:01:33,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:01:37,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:42,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:43,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:01:43,252 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:01:43,253 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:01:43,254 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:01:44,150 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:01:44,150 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:01:48,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:48,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:01:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:01:48,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:01:50,730 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:01:53,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:58,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:01:59,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:01:59,210 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:01:59,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:01:59,212 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:02:00,163 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:02:00,163 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:02:03,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:02:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:02:03,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:02:04,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:09,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:14,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:15,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:02:15,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:02:15,466 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:02:15,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:02:16,176 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:02:16,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:02:18,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:02:18,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:02:18,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:02:19,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:20,733 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:02:24,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:29,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:31,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:02:31,492 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:02:31,492 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:02:31,494 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:02:32,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:02:32,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:02:33,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:02:33,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:02:33,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:02:35,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:40,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:45,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:47,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:02:47,449 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:02:47,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:02:47,451 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:02:48,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:02:48,202 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:02:48,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:02:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:02:48,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:02:50,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:02:50,736 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:02:55,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:00,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:03,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:03:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:03:03,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:03:03,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:03:03,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:03:03,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:03:03,612 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:03:04,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:03:06,216 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:03:06,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:11,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:16,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:18,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:03:18,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:03:18,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:03:19,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:03:19,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:03:19,921 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:03:19,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:03:20,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:03:20,739 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:03:21,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:22,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:03:26,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:31,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:33,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:03:33,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:03:33,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:03:35,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:03:35,854 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:03:35,854 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:03:35,856 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:03:36,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:03:36,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:38,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:03:41,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:46,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:48,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:03:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:03:48,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:03:50,743 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:03:51,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:03:51,959 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:03:51,960 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:03:51,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:03:51,961 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:03:52,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:03:54,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:03:56,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:01,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:03,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:04:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:04:03,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:04:07,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:07,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:04:07,916 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:04:07,916 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:04:07,918 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:04:08,268 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:04:10,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:04:12,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:17,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:18,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:04:18,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:04:18,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:04:20,746 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:04:23,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:24,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:04:24,231 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:04:24,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:04:24,233 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:04:24,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:04:26,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:04:29,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:33,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:04:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:04:33,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:04:34,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:39,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:40,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:04:40,370 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:04:40,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:04:40,371 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:04:41,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:04:42,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:04:45,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:48,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:04:48,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:04:48,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:04:50,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:50,749 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:04:55,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:04:56,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:04:56,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:04:56,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:04:56,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:04:57,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:04:58,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:05:01,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:03,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:05:03,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:05:03,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:05:06,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:11,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:12,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:05:12,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:05:12,651 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:05:12,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:05:13,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:05:14,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:05:16,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:18,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:05:18,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:05:18,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:05:20,753 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:05:21,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:26,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:28,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:05:28,501 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:05:28,501 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:05:28,503 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:05:29,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:05:30,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:05:32,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:33,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:05:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:05:33,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:05:37,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:42,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:44,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:05:44,772 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:05:44,773 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:05:44,775 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:05:45,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:05:46,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:05:47,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:48,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:05:48,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:05:48,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:05:50,755 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:05:53,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:05:58,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:01,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:06:01,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:06:01,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:06:01,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:06:01,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:06:02,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:06:03,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:06:03,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:06:03,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:06:04,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:09,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:14,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:16,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:06:16,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:06:16,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:06:16,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:06:17,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:06:18,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:06:18,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:06:18,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:06:18,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:06:20,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:20,760 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:06:25,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:30,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:33,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:06:33,094 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:06:33,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:06:33,096 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:06:33,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:06:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:06:33,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:06:33,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:06:34,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:06:36,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:41,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:46,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:48,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:06:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:06:48,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:06:49,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:06:49,341 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:06:49,342 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:06:49,343 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:06:49,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:06:50,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:06:50,762 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:06:51,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:06:56,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:01,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:03,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:07:03,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:07:03,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:07:05,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:07:05,393 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:07:05,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:07:05,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:07:05,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:07:06,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:07:07,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:12,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:17,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:18,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:07:18,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:07:18,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:07:20,766 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:07:21,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:07:21,736 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:07:21,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:07:21,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:07:22,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:07:22,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:07:22,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:27,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:32,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:33,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:07:33,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:07:33,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:07:37,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:07:37,699 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:07:37,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:07:37,701 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:07:38,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:07:38,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:07:38,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:43,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:48,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:07:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:07:48,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:07:49,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:50,769 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:07:54,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:07:54,010 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:07:54,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:07:54,012 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:07:54,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:07:55,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:07:56,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:08:00,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:03,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:08:03,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:08:03,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:08:05,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:09,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:08:09,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:08:09,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:08:09,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:08:10,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:08:10,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:12,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:08:15,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:18,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:08:18,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:08:18,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:08:20,773 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:08:21,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:26,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:08:26,289 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:08:26,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:08:26,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:08:26,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:08:27,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:28,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:08:32,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:33,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:08:33,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:08:33,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:08:37,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:42,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:08:42,584 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:08:42,584 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:08:42,585 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:08:43,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:08:43,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:44,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:08:48,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:08:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:08:48,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:08:48,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:50,777 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:08:53,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:08:58,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:08:58,423 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:08:58,424 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:08:58,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:08:58,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:08:59,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:00,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:09:03,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:09:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:09:03,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:09:04,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:09,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:14,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:14,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:09:14,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:09:14,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:09:14,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:09:15,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:09:16,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:09:18,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:09:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:09:18,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:09:19,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:20,779 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:09:24,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:29,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:30,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:09:30,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:09:30,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:09:30,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:09:31,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:09:32,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:09:33,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:09:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:09:33,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:09:35,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:40,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:45,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:46,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:09:46,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:09:46,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:09:46,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:09:47,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:09:48,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:09:48,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:09:48,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:09:48,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:09:50,783 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:09:50,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:09:55,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:00,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:02,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:10:02,714 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:10:02,714 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:10:02,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:10:03,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:10:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:10:03,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:10:03,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:10:04,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:10:06,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:11,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:16,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:18,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:10:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:10:18,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:10:18,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:10:18,792 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:10:18,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:10:18,794 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:10:19,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:10:20,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:10:20,786 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:10:21,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:26,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:31,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:33,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:10:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:10:33,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:10:34,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:10:34,736 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:10:34,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:10:34,738 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:10:35,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:10:36,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:10:37,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:42,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:47,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:48,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:10:48,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:10:48,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:10:50,789 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:10:50,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:10:50,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:10:50,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:10:50,992 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:10:51,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:10:52,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:10:52,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:10:57,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:02,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:03,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:11:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:11:03,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:11:07,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:11:07,264 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:11:07,264 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:11:07,266 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:11:07,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:11:08,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:08,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:11:13,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:18,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:11:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:11:18,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:11:18,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:20,792 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:11:23,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:11:23,148 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:11:23,149 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:11:23,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:11:23,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:11:24,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:24,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:11:29,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:33,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:11:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:11:33,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:11:34,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:39,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:11:39,177 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:11:39,177 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:11:39,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:11:39,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:11:40,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:40,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:11:45,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:48,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:11:48,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:11:48,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:11:50,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:50,795 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:11:55,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:11:55,524 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:11:55,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:11:55,526 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:11:55,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:11:56,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:11:56,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:12:01,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:03,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:12:03,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:12:03,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:12:06,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:11,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:12:11,481 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:12:11,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:12:11,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:12:11,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:12:12,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:12,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:12:17,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:18,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:12:18,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:12:18,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:12:20,798 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:12:22,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:27,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:12:27,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:12:27,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:12:27,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:12:27,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:12:28,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:28,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:12:33,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:12:33,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:12:33,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:12:33,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:38,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:43,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:43,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:12:43,686 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:12:43,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:12:43,689 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:12:43,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:12:44,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:12:48,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:12:48,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:12:48,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:12:48,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:50,800 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:12:53,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:58,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:12:59,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:12:59,718 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:12:59,718 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:12:59,720 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:13:00,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:13:00,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:13:03,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:13:03,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:13:03,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:13:04,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:09,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:14,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:15,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:13:15,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:13:15,829 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:13:15,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:13:16,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:13:16,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:13:18,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:13:18,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:13:18,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:13:20,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:20,803 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:13:25,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:30,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:31,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:13:31,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:13:31,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:13:31,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:13:32,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:13:32,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:13:33,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:13:33,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:13:33,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:13:36,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:41,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:46,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:48,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:13:48,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:13:48,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:13:48,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:13:48,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:13:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:13:48,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:13:48,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:13:48,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:13:50,806 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:13:51,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:13:56,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:01,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:03,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:14:03,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:14:03,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:14:04,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:14:04,186 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:14:04,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:14:04,188 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:14:04,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:14:04,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:14:07,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:12,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:17,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:18,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:14:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:14:18,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:14:20,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:14:20,358 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:14:20,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:14:20,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:14:20,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:14:20,810 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:14:22,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:14:22,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:27,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:32,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:33,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:14:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:14:33,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:14:36,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:14:36,390 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:14:36,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:14:36,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:14:36,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:14:38,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:38,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:14:43,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:48,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:14:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:14:48,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:14:48,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:50,812 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:14:52,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:14:52,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:14:52,522 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:14:52,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:14:52,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:14:54,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:14:54,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:14:59,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:03,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:15:03,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:15:03,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:15:04,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:08,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:15:08,372 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:15:08,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:15:08,374 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:15:08,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:15:10,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:10,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:15:15,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:18,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:15:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:15:18,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:15:20,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:20,815 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:15:24,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:15:24,845 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:15:24,845 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:15:24,847 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:15:25,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:15:25,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:26,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:15:30,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:33,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:15:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:15:33,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:15:36,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:40,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:15:40,709 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:15:40,710 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:15:40,712 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:15:40,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:15:41,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:42,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:15:46,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:48,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:15:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:15:48,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:15:50,820 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:15:51,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:56,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:15:57,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:15:57,173 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:15:57,174 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:15:57,175 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:15:57,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:15:58,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:16:02,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:03,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:16:03,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:16:03,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:16:07,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:12,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:13,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:16:13,587 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:16:13,587 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:16:13,589 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:16:13,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:16:15,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:16:17,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:18,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:16:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:16:18,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:16:20,821 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:16:22,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:27,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:29,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:16:29,665 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:16:29,665 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:16:29,667 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:16:29,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:16:31,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:16:33,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:33,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:16:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:16:33,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:16:38,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:43,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:45,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:16:45,862 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:16:45,862 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:16:45,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:16:45,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:16:47,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:16:48,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:16:48,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:16:48,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:16:49,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:50,825 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:16:54,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:16:59,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:01,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:17:01,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:17:01,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:17:01,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:17:02,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:17:03,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:17:03,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:17:03,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:17:03,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:17:05,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:10,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:15,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:18,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:17:18,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:17:18,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:17:18,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:17:18,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:17:18,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:17:18,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:17:18,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:17:20,829 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:17:20,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:21,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:17:25,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:30,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:33,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:17:33,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:17:33,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:17:34,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:17:34,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:17:34,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:17:34,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:17:34,932 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:17:36,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:37,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:17:41,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:46,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:48,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:17:48,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:17:48,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:17:50,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:17:50,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:17:50,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:17:50,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:17:50,829 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:17:50,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:17:51,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:17:53,947 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:17:56,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:01,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:03,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:18:03,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:18:03,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:18:06,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:18:06,750 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:18:06,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:18:06,752 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:18:06,958 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:18:07,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:09,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:18:12,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:17,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:18,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:18:18,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:18:18,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:18:20,833 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:18:22,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:18:22,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:18:22,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:18:22,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:18:22,971 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:18:23,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:25,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:18:28,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:33,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:18:33,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:18:33,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:18:34,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:38,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:18:38,831 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:18:38,831 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:18:38,833 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:18:38,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:18:39,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:41,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:18:44,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:48,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:18:48,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:18:48,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:18:50,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:50,836 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:18:55,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:18:55,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:18:55,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:18:55,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:18:56,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:18:56,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:18:58,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:19:01,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:03,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:19:03,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:19:03,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:19:06,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:10,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:19:10,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:19:10,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:19:10,982 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:19:11,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:19:11,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:14,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:19:16,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:18,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:19:18,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:19:18,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:19:20,840 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:19:22,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:27,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:19:27,317 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:19:27,317 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:19:27,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:19:28,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:19:28,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:30,028 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:19:33,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:19:33,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:19:33,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:19:33,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:38,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:43,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:19:43,550 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:19:43,550 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:19:43,552 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:19:44,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:19:44,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:46,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:19:48,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:19:48,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:19:48,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:19:50,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:50,842 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:19:55,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:19:59,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:19:59,500 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:19:59,501 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:19:59,502 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:20:00,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:20:01,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:02,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:20:03,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:20:03,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:20:03,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:20:06,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:11,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:15,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:20:15,562 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:20:15,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:20:15,564 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:20:16,069 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:20:17,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:18,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:20:18,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:20:18,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:20:18,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:20:20,845 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:20:22,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:27,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:31,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:20:31,614 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:20:31,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:20:31,616 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:20:32,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:20:33,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:33,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:20:33,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:20:33,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:20:34,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:20:38,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:43,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:47,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:20:47,770 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:20:47,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:20:47,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:20:48,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:20:48,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:20:48,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:20:48,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:20:48,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:50,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:20:50,850 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:20:53,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:20:58,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:03,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:21:03,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:21:03,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:21:03,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:21:03,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:21:03,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:21:03,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:03,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:21:04,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:21:06,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:21:08,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:13,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:18,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:21:18,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:21:18,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:21:19,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:19,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:21:19,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:21:19,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:21:19,982 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:21:20,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:21:20,851 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:21:22,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:21:24,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:29,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:33,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:21:33,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:21:33,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:21:35,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:36,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:21:36,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:21:36,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:21:36,109 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:21:36,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:21:38,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:21:41,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:46,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:48,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:21:48,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:21:48,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:21:50,854 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:21:51,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:21:52,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:21:52,064 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:21:52,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:21:52,066 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:21:52,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:21:54,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:21:57,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:02,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:03,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:22:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:22:03,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:22:07,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:08,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:22:08,386 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:22:08,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:22:08,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:22:09,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:22:10,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:22:13,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:18,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:22:18,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:22:18,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:22:18,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:20,857 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:22:23,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:24,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:22:24,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:22:24,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:22:24,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:22:25,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:22:26,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:22:29,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:22:33,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:22:33,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:22:34,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:39,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:40,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:22:40,361 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:22:40,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:22:40,363 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:22:41,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:22:42,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:22:45,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:22:48,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:22:48,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:22:50,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:50,860 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:22:55,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:22:56,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:22:56,512 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:22:56,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:22:56,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:22:57,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:22:58,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:23:01,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:23:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:23:03,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:23:06,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:11,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:12,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:23:12,375 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:23:12,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:23:12,377 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:23:13,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:23:14,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:23:17,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:18,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:23:18,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:23:18,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:23:20,864 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:23:22,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:27,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:28,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:23:28,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:23:28,559 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:23:28,560 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:23:29,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:23:30,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:23:33,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:23:33,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:23:33,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:23:38,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:43,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:44,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:23:44,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:23:44,600 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:23:44,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:23:45,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:23:46,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:23:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:23:48,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:23:48,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:23:48,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:50,865 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:23:53,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:23:58,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:00,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:24:00,647 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:24:00,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:24:00,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:24:01,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:24:02,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:24:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:24:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:24:03,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:24:04,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:09,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:14,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:16,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:24:16,515 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:24:16,516 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:24:16,517 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:24:17,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:24:18,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:24:18,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:24:18,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:24:18,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:24:20,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:20,870 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:24:25,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:30,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:32,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:24:32,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:24:32,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:24:32,758 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:24:33,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:24:33,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:24:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:24:33,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:24:34,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:24:36,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:41,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:46,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:48,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:24:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:24:48,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:24:48,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:24:48,902 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:24:48,903 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:24:48,904 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:24:49,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:24:50,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:24:50,872 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:24:51,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:24:56,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:01,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:03,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:25:03,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:25:03,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:25:05,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:25:05,234 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:25:05,234 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:25:05,236 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:25:05,302 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:25:07,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:08,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:25:12,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:17,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:18,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:25:18,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:25:18,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:25:20,875 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:25:21,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:25:21,170 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:25:21,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:25:21,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:25:21,316 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:25:23,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:24,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:25:28,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:33,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:25:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:25:33,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:25:37,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:25:37,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:25:37,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:25:37,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:25:38,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:25:38,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:40,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:25:43,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:48,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:25:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:25:48,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:25:48,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:50,878 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:25:53,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:25:53,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:25:53,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:25:53,444 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:25:54,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:25:54,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:25:56,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:25:59,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:03,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:26:03,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:26:03,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:26:04,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:09,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:09,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:26:09,690 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:26:09,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:26:09,692 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:26:10,357 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:26:12,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:26:14,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:18,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:26:18,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:26:18,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:26:20,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:20,881 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:26:25,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:25,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:26:25,939 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:26:25,939 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:26:25,941 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:26:26,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:26:28,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:26:30,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:33,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:26:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:26:33,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:26:36,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:41,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:41,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:26:41,865 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:26:41,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:26:41,867 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:26:42,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:26:44,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:26:46,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:48,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:26:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:26:48,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:26:50,884 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:26:51,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:56,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:26:57,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:26:57,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:26:57,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:26:57,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:26:58,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:27:00,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:27:01,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:03,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:27:03,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:27:03,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:27:07,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:12,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:14,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:27:14,144 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:27:14,145 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:27:14,146 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:27:14,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:27:16,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:27:18,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:18,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:27:18,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:27:18,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:27:20,888 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:27:23,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:28,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:30,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:27:30,370 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:27:30,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:27:30,372 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:27:30,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:27:32,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:27:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:27:33,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:27:33,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:27:34,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:39,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:44,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:46,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:27:46,397 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:27:46,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:27:46,399 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:27:46,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:27:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:27:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:27:48,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:27:48,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:27:49,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:50,891 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:27:54,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:27:59,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:02,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:28:02,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:28:02,471 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:28:02,473 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:28:03,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:28:03,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:28:03,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:28:03,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:28:04,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:28:05,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:10,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:15,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:18,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:28:18,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:28:18,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:28:18,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:28:18,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:28:18,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:28:18,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:28:19,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:28:20,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:28:20,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:20,893 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:28:25,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:30,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:28:33,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:28:33,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:28:34,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:28:34,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:28:34,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:28:34,700 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:28:35,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:28:36,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:28:36,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:41,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:46,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:28:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:28:48,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:28:50,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:28:50,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:28:50,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:28:50,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:28:50,933 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:28:51,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:28:51,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:28:52,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:28:56,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:01,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:03,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:29:03,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:29:03,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:29:07,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:29:07,187 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:29:07,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:29:07,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:07,189 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:29:07,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:29:08,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:29:12,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:17,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:18,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:29:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:29:18,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:29:20,936 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:29:22,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:23,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:29:23,120 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:29:23,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:29:23,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:29:23,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:29:24,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:29:28,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:33,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:29:33,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:29:33,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:29:38,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:39,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:29:39,337 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:29:39,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:29:39,339 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:29:39,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:29:42,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:29:44,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:48,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:29:48,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:29:48,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:29:49,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:50,938 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:29:54,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:29:55,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:29:55,492 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:29:55,492 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:29:55,494 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:29:55,552 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:29:58,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:30:00,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:03,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:30:03,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:30:03,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:30:05,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:10,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:11,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:30:11,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:30:11,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:30:11,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:30:11,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:30:14,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:30:16,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:30:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:30:18,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:30:20,943 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:30:21,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:26,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:27,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:30:27,595 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:30:27,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:30:27,597 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:30:28,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:30:30,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:30:32,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:30:33,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:30:33,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:30:37,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:42,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:43,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:30:43,771 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:30:43,772 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:30:43,773 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:30:44,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:30:46,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:30:47,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:48,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:30:48,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:30:48,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:30:50,945 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:30:52,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:57,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:30:59,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:30:59,926 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:30:59,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:30:59,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:31:00,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:31:02,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:03,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:31:03,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:31:03,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:31:08,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:13,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:15,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:31:15,872 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:31:15,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:31:15,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:31:16,618 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:31:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:31:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:31:18,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:31:18,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:18,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:20,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:20,948 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:31:24,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:24,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:25,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:29,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:32,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:31:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:31:33,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:31:34,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:35,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:35,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:31:36,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:40,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:45,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:48,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:31:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:31:48,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:31:50,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:31:50,951 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:31:51,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:31:51,590 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:31:51,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:31:51,592 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:31:51,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:31:54,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:31:55,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:00,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:03,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:32:03,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:32:03,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:32:05,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:07,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:32:07,832 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:32:07,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:32:07,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:32:08,662 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:32:10,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:32:10,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:15,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:18,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:32:18,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:32:18,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:32:20,954 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:32:20,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:23,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:32:23,778 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:32:23,779 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:32:23,780 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:32:24,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:32:26,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:32:26,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:31,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:33,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:32:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:32:33,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:32:37,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:39,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:32:39,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:32:39,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:32:39,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:32:40,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:32:42,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:32:42,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:47,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:32:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:32:48,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:32:50,959 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:32:52,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:56,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:32:56,180 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:32:56,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:32:56,181 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:32:56,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:32:58,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:32:58,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:33:03,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:33:03,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:33:03,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:33:08,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:12,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:33:12,218 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:33:12,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:33:12,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:33:12,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:33:14,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:14,721 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:33:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:33:18,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:33:18,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:33:19,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:20,960 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:33:24,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:28,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:33:28,342 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:33:28,342 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:33:28,344 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:33:28,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:33:30,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:30,734 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:33:33,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:33:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:33:33,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:33:35,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:40,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:44,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:33:44,454 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:33:44,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:33:44,456 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:33:44,745 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:33:46,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:46,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:33:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:33:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:33:48,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:33:50,964 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:33:51,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:33:56,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:00,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:34:00,498 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:34:00,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:34:00,500 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:34:00,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:34:02,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:02,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:34:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:34:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:34:03,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:34:07,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:12,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:16,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:34:16,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:34:16,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:34:16,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:34:16,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:34:18,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:34:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:34:18,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:34:18,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:34:20,968 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:34:23,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:28,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:32,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:34:32,415 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:34:32,415 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:34:32,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:34:32,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:34:33,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:34:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:34:33,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:34:34,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:34,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:34:39,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:44,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:48,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:34:48,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:34:48,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:34:48,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:34:48,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:34:48,633 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:34:48,634 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:34:48,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:34:49,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:50,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:34:50,970 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:34:54,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:34:59,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:35:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:35:03,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:35:04,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:35:04,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:35:04,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:35:04,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:35:04,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:35:05,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:06,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:35:10,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:15,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:18,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:35:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:35:18,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:35:20,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:35:20,403 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:35:20,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:35:20,404 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:35:20,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:35:20,971 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:35:20,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:22,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:35:25,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:30,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:35:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:35:33,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:35:36,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:36,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:35:36,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:35:36,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:35:36,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:35:36,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:35:38,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:35:41,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:46,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:35:48,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:35:48,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:35:50,975 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:35:51,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:35:52,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:35:52,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:35:52,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:35:52,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:35:52,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:35:54,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:35:57,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:02,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:36:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:36:03,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:36:08,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:08,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:36:08,867 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:36:08,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:36:08,869 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:36:09,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:36:10,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:36:13,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:36:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:36:18,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:36:19,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:20,980 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:36:24,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:36:24,843 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:36:24,843 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:36:24,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:24,845 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:36:24,874 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:36:26,875 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:36:29,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:36:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:36:33,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:36:35,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:40,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:40,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:36:40,817 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:36:40,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:36:40,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:36:40,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:36:42,887 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:36:45,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:36:48,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:36:48,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:36:50,981 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:36:50,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:55,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:36:56,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:36:56,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:36:56,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:36:56,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:36:56,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:36:58,902 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:37:01,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:37:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:37:03,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:37:07,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:12,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:13,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:37:13,013 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:37:13,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:37:13,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:37:13,914 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:37:14,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:37:18,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:37:18,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:37:18,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:37:20,985 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:37:23,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:28,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:37:28,960 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:37:28,960 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:37:28,961 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:37:29,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:37:29,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:30,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:37:33,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:37:33,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:37:33,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:37:35,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:40,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:45,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:37:45,111 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:37:45,111 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:37:45,113 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:37:45,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:37:46,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:46,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:37:48,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:37:48,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:37:48,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:37:50,989 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:37:51,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:37:56,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:01,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:38:01,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:38:01,076 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:38:01,078 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:38:01,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:38:02,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:02,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:38:03,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:38:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:38:03,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:38:07,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:12,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:17,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:38:17,345 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:38:17,345 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:38:17,347 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:38:17,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:38:18,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:18,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:38:18,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:38:18,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:38:18,970 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:38:20,991 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:38:23,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:28,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:38:33,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:38:33,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:38:33,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:38:33,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:38:33,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:38:33,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:38:33,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:38:34,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:34,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:38:39,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:44,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:48,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:38:48,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:38:48,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:38:49,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:38:49,535 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:38:49,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:38:49,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:38:49,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:38:50,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:38:50,994 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:38:50,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:38:55,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:00,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:03,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:39:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:39:03,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:39:05,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:39:05,484 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:39:05,485 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:39:05,486 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:39:06,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:39:06,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:07,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:39:11,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:16,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:18,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:39:18,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:39:18,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:39:20,997 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:39:21,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:39:21,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:39:21,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:39:21,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:21,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:39:22,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:39:23,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:39:26,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:31,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:39:33,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:39:33,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:39:36,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:37,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:39:37,782 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:39:37,782 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:39:37,784 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:39:38,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:39:39,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:39:41,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:46,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:48,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:39:48,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:39:48,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:39:51,000 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:39:52,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:39:53,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:39:53,715 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:39:53,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:39:53,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:39:54,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:39:55,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:39:57,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:02,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:03,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:40:03,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:40:03,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:40:08,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:09,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:40:09,778 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:40:09,779 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:40:09,780 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:40:10,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:40:11,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:40:13,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:18,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:40:18,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:40:18,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:40:19,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:21,003 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:40:25,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:25,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:40:25,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:40:25,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:40:25,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:40:26,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:40:27,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:40:30,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:40:33,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:40:33,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:40:36,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:41,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:41,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:40:41,998 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:40:41,998 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:40:42,000 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:40:42,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:40:43,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:40:47,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:48,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:40:48,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:40:48,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:40:51,005 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:40:52,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:57,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:40:58,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:40:58,065 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:40:58,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:40:58,067 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:40:58,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:40:59,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:41:02,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:03,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:41:03,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:41:03,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:41:07,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:12,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:14,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:41:14,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:41:14,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:41:14,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:41:15,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:41:17,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:41:18,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:41:18,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:41:18,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:41:21,010 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:41:24,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:29,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:30,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:41:30,384 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:41:30,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:41:30,386 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:41:31,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:41:33,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:41:33,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:41:33,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:41:33,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:41:34,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:39,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:44,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:46,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:41:46,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:41:46,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:41:46,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:41:47,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:41:48,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:41:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:41:48,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:41:49,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:41:50,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:41:51,012 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:41:56,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:01,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:02,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:42:02,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:42:02,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:42:02,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:42:03,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:42:03,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:42:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:42:03,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:42:05,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:42:06,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:11,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:16,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:42:18,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:42:18,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:42:18,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:42:18,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:42:18,954 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:42:18,955 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:42:19,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:42:21,015 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:42:21,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:42:22,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:27,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:32,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:33,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:42:33,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:42:33,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:42:34,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:42:34,899 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:42:34,899 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:42:34,901 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:42:35,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:42:37,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:42:37,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:42,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:47,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:48,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:42:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:42:48,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:42:51,018 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:42:51,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:42:51,152 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:42:51,153 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:42:51,154 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:42:51,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:42:53,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:42:53,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:42:58,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:03,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:43:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:43:03,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:43:07,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:43:07,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:43:07,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:43:07,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:43:08,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:43:08,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:09,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:43:13,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:43:18,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:43:18,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:43:18,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:21,021 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:43:23,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:43:23,286 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:43:23,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:43:23,288 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:43:24,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:43:24,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:25,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:43:29,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:33,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:43:33,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:43:33,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:43:34,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:39,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:43:39,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:43:39,419 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:43:39,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:43:40,230 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:43:40,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:41,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:43:45,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:43:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:43:48,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:43:50,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:51,024 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:43:55,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:43:55,630 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:43:55,630 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:43:55,632 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:43:56,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:43:56,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:43:57,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:44:01,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:44:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:44:03,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:44:07,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:11,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:44:11,573 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:44:11,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:44:11,575 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:44:12,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:44:13,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:44:13,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:18,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:44:18,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:44:18,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:44:18,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:21,027 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:44:24,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:27,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:44:27,415 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:44:27,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:44:27,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:44:28,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:44:29,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:44:29,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:33,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:44:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:44:33,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:44:34,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:39,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:43,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:44:43,458 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:44:43,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:44:43,460 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:44:44,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:44:45,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:44:45,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:44:48,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:44:48,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:44:50,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:51,030 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:44:56,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:44:59,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:44:59,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:44:59,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:44:59,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:45:00,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:45:01,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:45:01,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:03,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:45:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:45:03,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:45:06,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:11,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:15,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:45:15,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:45:15,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:45:15,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:45:16,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:45:17,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:45:17,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:18,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:45:18,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:45:18,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:45:21,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:45:23,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:28,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:31,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:45:31,216 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:45:31,217 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:45:31,218 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:45:31,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:45:33,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:33,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:45:33,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:45:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:45:33,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:45:38,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:43,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:47,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:45:47,456 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:45:47,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:45:47,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:45:48,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:45:48,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:45:48,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:45:48,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:45:49,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:45:49,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:45:51,036 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:45:55,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:00,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:03,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:46:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:46:03,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:46:03,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:46:03,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:46:03,625 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:46:03,627 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:46:04,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:46:05,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:46:05,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:10,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:15,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:18,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:46:18,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:46:18,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:46:19,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:46:19,490 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:46:19,490 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:46:19,492 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:46:20,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:46:21,040 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:46:21,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:21,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:46:26,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:31,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:46:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:46:33,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:46:35,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:46:35,582 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:46:35,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:46:35,584 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:46:36,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:46:36,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:37,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:46:41,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:46,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:48,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:46:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:46:48,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:46:51,042 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:46:51,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:46:51,533 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:46:51,534 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:46:51,535 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:46:52,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:46:52,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:46:53,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:46:57,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:02,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:03,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:47:03,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:47:03,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:47:07,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:47:07,474 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:47:07,474 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:47:07,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:47:08,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:47:08,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:09,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:47:13,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:18,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:47:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:47:18,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:47:18,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:21,045 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:47:23,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:47:23,656 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:47:23,656 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:47:23,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:23,658 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:47:24,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:47:25,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:47:28,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:47:33,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:47:33,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:47:34,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:39,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:39,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:47:39,600 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:47:39,600 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:47:39,602 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:47:40,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:47:41,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:47:44,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:48,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:47:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:47:48,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:47:50,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:51,048 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:47:55,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:47:55,771 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:47:55,771 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:47:55,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:47:55,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:47:56,437 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:47:57,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:48:00,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:03,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:48:03,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:48:03,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:48:06,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:11,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:11,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:48:11,720 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:48:11,720 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:48:11,722 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:48:12,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:48:13,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:48:16,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:18,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:48:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:48:18,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:48:21,051 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:48:22,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:27,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:27,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:48:27,971 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:48:27,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:48:27,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:48:28,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:48:29,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:48:32,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:33,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:48:33,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:48:33,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:48:38,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:43,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:44,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:48:44,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:48:44,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:48:44,033 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:48:44,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:48:45,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:48:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:48:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:48:48,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:48:49,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:48:51,054 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:48:55,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:00,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:00,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:49:00,195 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:49:00,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:49:00,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:49:00,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:49:01,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:49:03,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:49:03,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:49:03,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:49:05,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:10,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:15,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:16,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:49:16,210 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:49:16,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:49:16,213 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:49:16,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:49:17,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:49:18,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:49:18,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:49:18,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:49:20,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:21,057 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:49:26,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:31,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:32,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:49:32,238 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:49:32,238 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:49:32,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:49:32,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:49:33,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:49:33,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:49:33,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:49:33,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:49:36,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:41,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:46,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:49:48,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:49:48,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:49:48,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:49:48,583 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:49:48,583 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:49:48,585 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:49:49,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:49:49,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:49:51,063 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:49:52,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:49:57,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:02,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:03,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:50:03,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:50:03,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:50:04,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:50:04,516 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:50:04,516 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:50:04,518 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:50:04,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:50:05,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:50:07,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:12,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:17,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:18,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:50:18,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:50:18,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:50:20,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:50:20,562 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:50:20,562 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:50:20,564 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:50:21,063 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:50:21,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:50:21,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:50:23,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:28,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:33,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:33,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:50:33,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:50:33,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:50:36,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:50:36,715 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:50:36,716 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:50:36,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:50:37,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:50:37,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:50:38,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:43,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:48,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:50:48,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:50:48,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:50:49,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:51,066 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:50:52,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:50:52,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:50:52,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:50:52,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:50:53,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:50:54,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:50:55,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:50:59,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:51:03,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:51:03,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:51:05,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:08,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:51:08,743 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:51:08,743 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:51:08,745 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:51:09,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:51:09,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:51:10,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:15,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:51:18,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:51:18,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:51:21,071 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:51:21,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:24,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:51:24,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:51:24,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:51:24,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:51:25,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:51:26,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:27,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:51:31,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:51:33,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:51:33,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:51:37,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:41,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:51:41,019 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:51:41,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:51:41,021 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:51:41,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:51:43,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:43,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:51:48,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:48,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:51:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:51:48,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:51:51,072 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:51:53,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:57,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:51:57,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:51:57,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:51:57,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:51:57,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:51:58,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:51:59,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:52:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:52:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:52:03,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:52:03,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:08,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:13,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:52:13,323 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:52:13,323 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:52:13,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:52:13,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:52:14,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:15,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:52:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:52:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:52:18,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:52:19,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:21,075 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:52:25,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:29,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:52:29,679 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:52:29,679 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:52:29,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:52:29,681 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:52:30,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:31,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:52:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:52:33,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:52:33,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:52:36,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:41,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:45,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:52:45,629 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:52:45,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:52:45,631 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:52:45,695 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:52:46,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:47,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:52:48,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:52:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:52:48,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:52:51,078 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:52:52,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:52:57,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:01,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:53:01,774 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:53:01,775 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:53:01,776 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:53:02,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:53:02,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:03,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:53:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:53:03,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:53:03,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:53:08,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:13,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:18,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:53:18,009 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:53:18,010 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:53:18,011 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:53:18,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:53:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:53:18,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:53:18,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:18,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:53:19,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:53:21,085 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:53:24,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:29,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:33,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:53:33,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:53:33,385 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:53:34,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:53:34,040 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:53:34,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:53:34,042 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:53:34,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:53:35,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:35,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:53:40,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:45,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:48,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:53:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:53:48,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:53:50,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:53:50,180 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:53:50,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:53:50,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:53:50,182 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:53:50,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:53:51,084 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:53:51,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:53:56,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:01,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:03,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:54:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:54:03,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:54:06,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:54:06,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:54:06,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:54:06,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:06,290 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:54:06,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:54:07,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:54:11,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:16,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:18,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:54:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:54:18,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:54:21,087 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:54:22,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:22,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:54:22,324 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:54:22,324 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:54:22,326 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:54:22,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:54:23,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:54:27,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:32,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:33,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:54:33,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:54:33,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:54:37,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:38,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:54:38,185 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:54:38,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:54:38,187 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:54:38,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:54:39,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:54:43,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:48,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:48,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:54:48,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:54:48,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:54:51,091 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:54:54,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:54:54,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:54:54,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:54:54,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:54:54,249 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:54:54,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:54:55,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:54:59,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:03,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:55:03,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:55:03,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:55:04,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:09,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:10,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:55:10,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:55:10,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:55:10,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:55:10,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:55:11,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:55:15,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:55:18,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:55:18,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:55:20,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:21,093 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:55:26,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:26,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:55:26,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:55:26,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:55:26,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:55:26,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:55:27,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:55:31,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:55:33,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:55:33,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:55:36,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:41,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:42,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:55:42,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:55:42,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:55:42,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:55:42,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:55:43,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:55:47,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:55:48,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:55:48,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:55:51,096 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:55:52,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:57,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:55:58,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:55:58,182 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:55:58,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:55:58,184 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:55:58,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:55:59,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:56:02,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:03,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:56:03,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:56:03,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:56:07,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:12,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:14,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:56:14,267 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:56:14,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:56:14,270 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:56:14,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:56:15,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:56:18,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:56:18,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:56:18,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:56:21,099 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:56:24,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:29,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:30,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:56:30,335 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:56:30,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:56:30,337 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:56:30,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:56:31,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:56:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:56:33,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:56:33,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:56:34,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:39,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:44,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:46,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:56:46,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:56:46,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:56:46,313 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:56:46,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:56:47,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:56:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:56:48,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:56:48,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:56:49,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:56:51,103 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:56:55,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:00,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:02,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:57:02,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:57:02,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:57:02,376 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:57:02,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:57:03,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:57:03,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:57:03,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:57:03,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:57:05,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:10,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:15,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:18,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:57:18,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:57:18,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:57:18,227 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:57:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:57:18,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:57:18,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:57:18,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:57:19,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:57:21,106 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:57:21,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:26,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:31,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:57:33,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:57:33,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:57:34,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:57:34,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:57:34,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:57:34,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:57:34,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:57:35,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:57:36,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:41,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:46,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:48,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:57:48,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:57:48,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:57:50,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:57:50,464 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:57:50,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:57:50,465 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:57:50,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:57:51,109 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:57:51,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:57:52,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:57:57,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:02,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:03,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:58:03,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:58:03,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:58:06,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:58:06,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:58:06,707 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:58:06,709 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:58:06,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:58:07,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:07,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:58:12,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:17,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:18,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:58:18,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:58:18,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:58:21,114 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:58:22,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:58:22,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:58:22,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:58:22,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:58:22,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:58:23,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:23,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:58:28,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:33,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:58:33,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:58:33,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:58:33,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:38,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:38,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:58:38,814 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:58:38,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:58:38,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:58:39,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:58:40,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:58:43,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:48,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:58:48,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:58:48,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:58:49,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:51,116 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:58:54,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:58:54,769 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:58:54,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:58:54,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:58:54,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:58:55,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:58:56,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:58:59,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:03,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:59:03,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:59:03,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:59:05,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:10,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:11,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:59:11,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:59:11,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:59:11,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:59:12,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:59:12,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:59:16,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:18,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:59:18,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:59:18,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:59:21,119 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:59:22,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:27,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:59:27,083 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:59:27,083 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:59:27,085 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:59:28,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:59:28,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:59:28,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:33,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:33,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:59:33,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:59:33,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:59:38,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:43,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:59:43,031 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:59:43,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:59:43,034 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 06:59:43,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 06:59:44,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:44,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 06:59:48,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 06:59:48,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 06:59:48,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 06:59:49,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:51,122 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 06:59:55,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 06:59:59,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 06:59:59,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 06:59:59,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 06:59:59,374 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:00:00,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:00:00,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:02,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:00:03,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:00:03,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:00:03,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:00:05,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:10,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:15,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:00:15,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:00:15,303 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:00:15,305 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:00:16,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:00:16,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:00:16,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:18,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:00:18,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:00:18,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:00:21,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:00:22,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:27,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:31,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:00:31,464 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:00:31,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:00:31,466 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:00:32,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:00:32,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:33,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:00:33,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:00:33,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:00:34,111 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:00:37,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:42,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:47,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:00:47,539 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:00:47,540 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:00:47,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:00:48,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:00:48,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:00:48,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:48,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:00:48,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:00:48,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:00:51,128 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:00:54,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:00:59,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:03,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:01:03,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:01:03,388 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:01:03,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:01:03,678 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:01:03,679 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:01:03,680 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:01:04,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:01:04,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:01:04,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:09,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:14,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:18,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:01:18,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:01:18,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:01:19,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:01:19,634 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:01:19,634 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:01:19,636 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:01:20,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:01:20,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:01:20,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:21,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:01:26,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:31,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:33,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:01:33,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:01:33,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:01:35,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:01:35,996 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:01:35,997 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:01:35,998 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:01:36,169 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:01:36,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:38,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:01:42,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:47,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:48,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:01:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:01:48,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:01:51,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:01:52,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:01:52,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:01:52,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:01:52,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:01:52,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:01:52,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:01:54,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:01:57,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:02,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:02:03,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:02:03,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:02:07,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:08,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:02:08,160 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:02:08,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:02:08,162 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:02:08,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:02:10,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:02:13,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:18,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:18,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:02:18,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:02:18,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:02:21,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:02:24,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:24,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:02:24,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:02:24,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:02:24,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:02:24,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:02:26,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:02:29,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:33,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:02:33,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:02:33,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:02:34,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:39,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:40,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:02:40,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:02:40,580 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:02:40,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:02:41,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:02:42,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:02:45,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:48,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:02:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:02:48,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:02:50,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:51,140 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:02:56,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:02:56,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:02:56,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:02:56,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:02:56,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:02:57,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:02:58,239 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:03:01,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:03:03,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:03:03,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:03:07,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:12,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:12,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:03:12,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:03:12,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:03:12,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:03:13,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:03:14,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:03:17,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:18,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:03:18,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:03:18,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:03:21,144 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:03:23,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:28,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:28,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:03:28,984 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:03:28,985 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:03:28,986 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:03:29,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:03:30,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:03:33,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:33,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:03:33,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:03:33,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:03:38,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:43,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:44,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:03:44,908 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:03:44,909 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:03:44,910 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:03:45,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:03:46,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:03:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:03:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:03:48,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:03:49,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:03:51,146 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:03:55,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:00,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:01,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:04:01,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:04:01,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:04:01,034 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:04:01,296 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:04:02,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:04:03,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:04:03,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:04:03,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:04:05,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:10,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:15,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:17,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:04:17,066 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:04:17,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:04:17,068 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:04:17,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:04:18,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:04:18,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:04:18,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:04:18,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:04:20,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:21,150 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:04:26,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:31,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:33,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:04:33,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:04:33,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:04:33,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:04:33,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:04:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:04:33,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:04:33,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:04:34,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:04:36,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:41,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:46,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:04:48,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:04:48,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:04:48,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:04:48,884 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:04:48,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:04:48,886 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:04:49,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:04:50,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:04:51,153 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:04:52,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:04:57,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:02,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:03,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:05:03,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:05:03,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:05:04,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:05:04,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:05:04,850 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:05:04,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:05:05,357 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:05:06,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:05:07,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:12,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:17,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:18,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:05:18,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:05:18,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:05:20,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:05:20,906 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:05:20,906 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:05:20,908 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:05:21,156 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:05:21,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:05:22,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:05:23,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:28,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:33,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:05:33,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:05:33,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:05:36,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:05:36,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:05:36,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:05:36,863 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:05:37,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:05:38,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:05:38,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:43,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:05:48,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:05:48,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:05:49,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:51,159 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:05:52,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:05:52,714 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:05:52,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:05:52,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:05:53,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:05:54,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:05:54,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:05:59,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:03,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:06:03,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:06:03,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:06:05,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:08,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:06:08,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:06:08,762 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:06:08,764 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:06:09,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:06:10,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:06:10,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:15,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:18,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:06:18,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:06:18,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:06:21,162 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:06:21,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:24,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:06:24,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:06:24,942 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:06:24,943 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:06:25,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:06:26,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:06:26,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:31,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:06:33,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:06:33,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:06:37,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:40,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:06:40,987 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:06:40,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:06:40,989 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:06:41,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:06:42,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:06:42,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:47,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:48,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:06:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:06:48,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:06:51,166 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:06:53,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:06:57,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:06:57,054 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:06:57,054 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:06:57,056 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:06:57,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:06:58,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:06:59,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:03,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:07:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:07:03,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:07:04,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:09,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:12,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:07:12,926 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:07:12,926 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:07:12,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:07:13,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:07:14,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:07:14,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:18,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:07:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:07:18,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:07:20,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:21,170 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:07:26,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:28,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:07:28,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:07:28,777 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:07:28,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:07:29,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:07:30,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:07:31,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:33,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:07:33,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:07:33,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:07:37,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:42,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:45,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:07:45,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:07:45,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:07:45,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:07:45,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:07:46,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:07:48,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:07:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:07:48,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:07:51,172 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:07:53,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:07:58,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:01,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:08:01,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:08:01,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:08:01,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:08:01,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:08:02,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:08:03,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:03,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:08:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:08:03,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:08:08,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:13,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:17,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:08:17,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:08:17,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:08:17,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:08:17,532 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:08:18,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:08:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:08:18,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:08:18,533 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:08:18,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:21,176 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:08:24,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:29,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:33,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:08:33,304 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:08:33,304 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:08:33,305 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:08:33,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:08:33,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:08:33,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:08:33,546 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:08:34,547 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:08:34,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:39,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:44,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:08:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:08:48,391 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:08:49,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:08:49,460 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:08:49,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:08:49,462 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:08:49,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:08:50,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:08:50,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:08:51,177 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:08:56,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:01,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:03,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:09:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:09:03,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:09:05,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:09:05,506 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:09:05,506 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:09:05,507 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:09:05,574 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:09:06,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:06,575 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:09:11,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:16,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:18,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:09:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:09:18,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:09:21,181 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:09:21,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:09:21,756 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:09:21,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:09:21,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:21,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:09:22,589 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:09:22,589 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:09:26,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:31,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:33,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:09:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:09:33,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:09:37,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:37,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:09:37,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:09:37,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:09:37,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:09:38,603 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:09:38,603 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:09:42,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:47,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:48,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:09:48,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:09:48,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:09:51,183 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:09:53,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:09:53,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:09:53,887 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:09:53,887 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:09:53,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:09:54,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:09:54,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:09:58,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:03,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:10:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:10:03,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:10:04,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:09,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:10,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:10:10,295 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:10:10,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:10:10,297 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:10:10,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:10:10,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:10:15,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:10:18,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:10:18,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:10:20,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:21,188 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:10:26,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:26,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:10:26,229 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:10:26,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:10:26,231 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:10:26,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:10:26,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:10:31,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:33,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:10:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:10:33,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:10:36,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:41,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:42,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:10:42,387 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:10:42,387 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:10:42,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:10:42,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:10:42,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:10:47,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:48,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:10:48,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:10:48,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:10:51,197 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:10:53,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:58,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:10:58,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:10:58,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:10:58,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:10:58,453 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:10:58,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:10:58,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:11:03,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:03,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:11:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:11:03,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:11:08,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:13,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:14,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:11:14,591 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:11:14,591 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:11:14,593 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:11:14,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:11:14,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:11:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:11:18,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:11:18,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:11:19,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:21,200 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:11:25,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:30,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:30,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:11:30,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:11:30,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:11:30,634 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:11:30,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:11:30,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:11:33,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:11:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:11:33,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:11:35,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:40,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:45,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:46,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:11:46,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:11:46,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:11:46,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:11:47,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:11:48,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:11:48,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:11:48,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:11:48,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:11:50,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:11:51,204 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:11:56,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:01,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:02,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:12:02,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:12:02,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:12:02,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:12:03,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:12:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:12:03,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:12:03,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:12:04,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:12:06,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:11,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:16,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:12:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:12:18,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:12:18,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:12:18,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:12:18,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:12:18,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:12:19,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:12:20,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:12:21,206 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:12:22,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:27,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:32,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:12:33,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:12:33,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:12:35,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:12:35,238 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:12:35,239 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:12:35,240 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:12:35,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:12:36,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:12:37,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:42,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:47,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:12:48,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:12:48,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:12:51,210 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:12:51,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:12:51,382 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:12:51,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:12:51,384 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:12:51,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:12:52,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:12:52,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:12:57,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:02,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:03,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:13:03,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:13:03,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:13:07,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:13:07,325 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:13:07,326 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:13:07,327 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:13:07,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:13:08,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:08,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:13:13,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:18,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:13:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:13:18,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:13:18,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:21,213 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:13:23,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:13:23,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:13:23,355 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:13:23,356 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:13:23,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:13:24,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:24,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:13:29,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:13:33,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:13:33,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:13:34,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:39,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:13:39,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:13:39,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:13:39,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:13:39,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:13:40,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:40,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:13:45,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:13:48,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:13:48,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:13:50,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:51,216 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:13:55,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:13:55,521 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:13:55,521 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:13:55,523 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:13:55,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:13:56,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:13:57,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:14:01,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:03,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:14:03,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:14:03,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:14:06,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:11,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:11,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:14:11,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:14:11,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:14:11,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:14:11,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:14:13,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:14:16,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:18,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:14:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:14:18,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:14:21,219 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:14:22,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:27,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:27,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:14:27,778 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:14:27,778 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:14:27,780 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:14:27,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:14:29,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:14:32,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:33,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:14:33,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:14:33,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:14:38,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:43,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:43,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:14:43,711 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:14:43,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:14:43,713 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:14:43,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:14:45,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:14:48,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:14:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:14:48,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:14:49,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:51,221 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:14:55,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:14:59,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:14:59,568 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:14:59,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:14:59,570 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:14:59,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:15:00,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:01,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:15:03,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:15:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:15:03,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:15:05,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:10,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:15,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:15:15,530 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:15:15,530 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:15:15,532 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:15:15,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:15:16,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:17,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:15:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:15:18,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:15:18,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:15:21,226 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:15:22,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:27,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:31,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:15:31,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:15:31,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:15:31,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:15:31,910 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:15:32,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:15:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:15:33,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:15:33,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:15:37,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:42,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:47,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:15:47,532 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:15:47,532 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:15:47,533 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:15:47,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:15:48,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:48,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:15:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:15:48,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:15:49,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:15:51,228 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:15:54,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:15:59,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:16:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:16:03,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:16:03,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:16:03,579 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:16:03,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:16:03,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:16:03,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:16:04,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:05,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:16:09,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:14,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:16:18,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:16:18,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:16:19,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:16:19,455 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:16:19,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:16:19,457 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:16:19,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:16:20,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:21,232 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:16:21,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:16:26,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:31,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:16:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:16:33,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:16:35,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:16:35,695 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:16:35,696 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:16:35,697 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:16:35,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:16:36,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:37,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:16:41,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:46,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:48,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:16:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:16:48,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:16:51,234 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:16:51,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:16:51,840 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:16:51,841 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:16:51,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:16:51,842 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:16:51,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:16:53,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:16:56,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:01,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:17:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:17:03,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:17:07,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:07,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:17:07,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:17:07,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:17:07,683 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:17:07,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:17:09,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:17:12,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:17,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:18,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:17:18,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:17:18,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:17:21,237 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:17:23,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:23,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:17:23,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:17:23,763 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:17:23,765 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:17:24,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:17:26,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:17:28,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:33,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:17:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:17:33,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:17:34,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:39,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:39,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:17:39,733 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:17:39,734 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:17:39,735 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:17:40,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:17:42,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:17:44,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:48,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:17:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:17:48,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:17:50,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:51,240 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:17:55,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:17:55,902 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:17:55,902 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:17:55,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:17:55,904 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:17:56,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:17:58,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:18:00,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:18:03,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:18:03,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:18:06,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:11,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:12,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:18:12,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:18:12,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:18:12,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:18:13,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:18:14,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:18:17,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:18,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:18:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:18:18,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:18:21,243 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:18:22,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:27,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:28,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:18:28,045 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:18:28,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:18:28,047 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:18:28,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:18:30,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:18:33,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:33,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:18:33,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:18:33,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:18:38,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:43,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:44,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:18:44,300 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:18:44,301 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:18:44,302 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:18:45,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:18:46,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:18:48,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:18:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:18:48,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:18:49,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:18:51,248 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:18:55,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:00,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:00,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:19:00,346 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:19:00,346 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:19:00,347 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:19:01,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:19:02,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:19:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:19:03,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:19:03,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:19:05,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:10,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:15,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:16,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:19:16,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:19:16,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:19:16,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:19:17,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:19:18,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:19:18,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:19:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:19:18,395 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:19:20,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:21,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:19:26,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:31,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:32,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:19:32,618 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:19:32,619 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:19:32,620 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:19:33,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:19:33,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:19:33,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:19:33,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:19:34,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:19:36,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:41,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:46,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:48,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:19:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:19:48,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:19:48,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:19:48,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:19:48,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:19:48,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:19:49,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:19:50,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:19:51,286 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:19:52,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:19:57,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:02,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:03,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:20:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:20:03,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:20:04,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:20:04,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:20:04,614 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:20:04,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:20:05,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:20:06,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:20:07,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:12,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:17,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:18,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:20:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:20:18,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:20:20,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:20:20,984 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:20:20,985 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:20:20,986 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:20:21,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:20:21,289 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:20:22,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:20:23,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:28,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:33,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:20:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:20:33,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:20:33,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:36,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:20:36,832 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:20:36,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:20:36,834 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:20:37,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:20:38,176 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:20:38,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:43,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:48,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:20:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:20:48,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:20:49,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:20:51,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:20:53,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:20:53,087 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:20:53,087 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:20:53,088 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:20:53,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:20:54,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:20:55,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:00,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:03,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:21:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:21:03,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:21:05,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:09,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:21:09,240 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:21:09,240 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:21:09,242 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:21:10,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:21:10,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:21:11,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:16,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:18,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:21:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:21:18,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:21:21,296 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:21:21,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:25,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:21:25,524 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:21:25,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:21:25,526 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:21:26,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:21:26,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:28,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:21:31,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:33,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:21:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:21:33,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:21:36,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:41,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:21:41,573 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:21:41,573 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:21:41,575 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:21:42,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:21:42,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:44,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:21:47,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:48,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:21:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:21:48,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:21:51,299 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:21:53,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:21:57,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:21:57,697 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:21:57,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:21:57,699 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:21:58,247 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:21:58,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:00,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:22:03,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:22:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:22:03,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:22:04,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:09,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:13,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:22:13,977 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:22:13,977 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:22:13,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:22:14,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:22:14,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:16,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:22:18,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:22:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:22:18,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:22:20,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:21,304 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:22:26,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:30,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:22:30,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:22:30,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:22:30,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:22:30,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:22:32,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:32,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:22:33,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:22:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:22:33,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:22:37,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:42,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:46,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:22:46,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:22:46,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:22:46,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:22:46,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:22:48,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:48,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:22:48,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:22:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:22:48,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:22:51,305 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:22:53,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:22:58,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:02,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:23:02,201 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:23:02,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:23:02,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:23:02,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:23:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:23:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:23:03,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:23:03,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:04,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:23:08,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:13,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:18,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:23:18,265 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:23:18,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:23:18,266 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:23:18,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:23:18,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:23:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:23:18,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:23:18,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:20,322 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:23:21,308 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:23:24,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:29,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:33,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:23:33,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:23:33,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:23:34,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:23:34,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:23:34,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:23:34,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:34,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:23:34,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:23:36,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:23:39,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:44,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:23:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:23:48,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:23:49,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:23:50,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:23:50,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:23:50,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:23:50,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:23:51,312 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:23:51,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:23:52,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:23:55,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:00,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:24:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:24:03,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:24:05,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:06,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:24:06,971 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:24:06,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:24:06,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:24:07,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:24:08,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:24:10,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:15,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:18,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:24:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:24:18,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:24:21,314 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:24:21,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:23,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:24:23,125 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:24:23,125 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:24:23,127 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:24:23,380 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:24:24,381 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:24:27,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:32,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:33,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:24:33,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:24:33,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:24:37,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:39,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:24:39,259 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:24:39,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:24:39,260 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:24:39,393 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:24:40,394 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:24:43,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:48,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:24:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:24:48,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:24:51,317 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:24:53,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:24:55,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:24:55,116 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:24:55,116 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:24:55,118 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:24:55,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:24:56,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:24:59,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:25:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:25:03,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:25:04,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:09,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:11,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:25:11,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:25:11,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:25:11,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:25:11,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:25:12,422 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:15,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:16,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:18,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:25:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:25:18,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:25:20,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:21,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:21,322 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:25:21,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:26,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:26,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:28,435 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:30,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:31,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:25:32,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:32,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:25:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:25:33,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:25:37,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:42,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:47,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:25:47,370 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:25:47,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:25:47,372 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:25:47,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:25:48,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:48,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:25:48,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:25:48,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:25:48,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:25:51,322 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:25:53,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:25:58,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:26:03,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:26:03,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:26:03,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:03,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:26:03,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:26:03,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:26:03,727 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:26:04,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:26:06,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:26:08,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:13,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:18,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:26:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:26:18,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:26:19,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:19,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:26:19,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:26:19,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:26:19,794 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:26:20,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:26:21,326 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:26:22,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:26:25,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:30,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:26:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:26:33,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:26:35,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:36,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:26:36,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:26:36,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:26:36,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:26:36,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:26:38,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:26:41,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:46,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:48,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:26:48,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:26:48,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:26:51,328 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:26:51,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:26:52,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:26:52,451 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:26:52,451 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:26:52,453 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:26:52,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:26:54,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:26:56,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:01,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:03,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:27:03,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:27:03,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:27:06,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:08,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:27:08,429 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:27:08,430 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:27:08,431 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:27:08,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:27:10,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:27:12,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:17,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:18,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:27:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:27:18,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:27:21,333 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:27:23,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:24,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:27:24,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:27:24,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:27:24,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:27:25,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:27:26,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:27:28,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:33,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:27:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:27:33,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:27:34,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:39,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:40,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:27:40,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:27:40,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:27:40,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:27:41,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:27:42,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:27:44,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:48,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:27:48,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:27:48,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:27:50,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:51,334 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:27:56,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:27:56,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:27:56,837 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:27:56,838 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:27:56,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:27:57,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:27:58,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:28:01,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:03,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:28:03,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:28:03,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:28:07,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:12,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:12,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:28:12,796 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:28:12,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:28:12,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:28:13,579 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:28:14,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:28:17,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:28:18,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:28:18,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:28:21,338 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:28:23,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:28,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:28,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:28:28,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:28:28,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:28:28,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:28:29,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:28:30,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:28:33,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:28:33,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:28:33,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:28:35,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:40,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:44,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:28:44,718 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:28:44,719 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:28:44,720 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:28:45,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:28:45,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:46,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:28:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:28:48,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:28:48,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:28:51,342 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:28:51,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:28:56,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:00,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:29:00,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:29:00,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:29:00,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:29:00,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:29:01,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:02,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:29:03,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:29:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:29:03,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:29:06,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:11,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:16,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:16,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:29:16,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:29:16,731 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:29:16,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:29:17,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:29:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:29:18,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:29:18,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:29:18,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:29:21,344 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:29:22,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:27,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:32,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:32,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:29:32,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:29:32,593 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:29:32,594 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:29:32,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:29:33,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:29:33,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:29:33,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:29:34,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:29:37,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:42,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:47,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:29:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:29:48,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:29:48,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:29:48,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:29:48,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:29:48,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:29:49,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:29:50,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:29:51,347 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:29:53,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:29:58,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:03,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:30:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:30:03,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:30:03,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:04,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:30:04,720 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:30:04,720 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:30:04,722 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:30:05,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:30:06,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:30:08,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:13,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:30:18,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:30:18,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:30:19,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:20,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:30:20,770 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:30:20,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:30:20,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:30:21,350 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:30:21,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:30:22,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:30:25,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:30,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:33,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:30:33,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:30:33,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:30:35,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:36,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:30:36,860 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:30:36,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:30:36,862 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:30:37,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:30:38,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:30:40,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:45,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:48,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:30:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:30:48,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:30:51,353 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:30:51,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:30:52,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:30:52,922 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:30:52,923 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:30:52,924 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:30:53,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:30:54,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:30:56,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:01,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:03,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:31:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:31:03,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:31:07,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:08,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:31:08,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:31:08,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:31:08,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:31:09,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:31:10,734 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:31:12,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:17,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:18,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:31:18,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:31:18,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:31:21,357 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:31:23,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:25,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:31:25,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:31:25,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:31:25,109 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:31:25,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:31:26,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:31:29,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:33,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:31:33,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:31:33,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:31:34,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:39,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:41,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:31:41,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:31:41,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:31:41,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:31:41,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:31:42,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:31:45,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:48,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:31:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:31:48,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:31:50,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:51,360 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:31:56,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:31:57,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:31:57,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:31:57,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:31:57,294 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:31:57,776 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:31:58,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:32:02,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:03,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:32:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:32:03,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:32:07,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:12,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:13,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:32:13,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:32:13,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:32:13,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:32:13,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:32:14,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:32:18,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:18,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:32:18,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:32:18,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:32:21,364 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:32:23,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:28,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:29,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:32:29,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:32:29,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:32:29,248 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:32:29,807 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:32:30,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:32:33,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:32:33,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:32:33,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:32:33,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:38,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:43,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:45,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:32:45,529 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:32:45,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:32:45,531 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:32:45,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:32:46,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:32:48,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:32:48,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:32:48,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:32:49,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:32:51,367 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:32:55,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:00,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:01,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:33:01,388 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:33:01,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:33:01,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:33:01,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:33:02,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:33:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:33:03,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:33:03,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:33:05,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:10,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:15,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:17,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:33:17,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:33:17,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:33:17,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:33:17,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:33:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:33:18,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:33:18,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:33:18,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:33:20,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:21,370 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:33:26,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:31,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:33,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:33:33,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:33:33,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:33:33,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:33:33,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:33:33,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:33:33,684 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:33:33,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:33:34,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:33:36,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:41,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:46,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:33:48,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:33:48,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:33:49,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:33:49,635 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:33:49,636 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:33:49,637 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:33:49,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:33:50,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:33:51,374 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:33:52,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:33:57,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:02,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:34:03,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:34:03,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:34:05,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:34:05,667 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:34:05,667 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:34:05,669 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:34:05,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:34:06,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:34:07,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:12,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:17,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:34:18,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:34:18,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:34:21,376 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:34:21,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:34:21,937 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:34:21,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:34:21,939 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:34:22,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:34:22,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:34:22,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:27,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:32,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:33,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:34:33,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:34:33,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:34:38,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:34:38,105 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:34:38,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:34:38,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:38,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:34:38,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:34:38,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:34:43,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:48,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:34:48,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:34:48,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:34:51,379 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:34:53,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:34:54,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:34:54,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:34:54,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:34:54,256 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:34:54,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:34:56,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:34:59,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:35:03,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:35:03,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:35:04,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:09,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:10,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:35:10,294 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:35:10,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:35:10,296 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:35:10,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:35:12,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:35:15,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:18,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:35:18,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:35:18,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:35:20,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:21,383 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:35:26,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:26,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:35:26,673 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:35:26,673 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:35:26,675 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:35:26,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:35:28,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:35:31,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:33,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:35:33,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:35:33,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:35:37,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:42,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:42,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:35:42,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:35:42,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:35:42,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:35:42,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:35:44,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:35:47,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:48,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:35:48,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:35:48,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:35:51,387 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:35:53,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:58,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:35:58,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:35:58,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:35:58,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:35:58,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:35:58,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:36:00,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:36:03,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:36:03,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:36:03,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:36:03,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:08,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:13,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:15,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:36:15,378 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:36:15,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:36:15,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:36:16,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:36:17,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:36:18,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:36:18,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:36:18,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:36:18,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:21,389 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:36:24,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:29,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:31,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:36:31,229 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:36:31,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:36:31,231 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:36:32,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:36:33,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:36:33,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:36:33,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:36:33,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:36:34,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:39,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:44,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:47,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:36:47,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:36:47,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:36:47,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:36:48,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:36:48,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:36:48,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:36:48,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:36:49,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:36:50,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:36:51,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:36:56,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:01,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:03,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:37:03,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:37:03,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:37:03,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:37:03,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:37:03,633 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:37:03,634 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:37:04,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:37:05,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:37:06,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:11,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:16,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:18,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:37:18,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:37:18,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:37:19,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:37:19,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:37:19,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:37:19,608 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:37:20,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:37:21,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:37:21,395 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:37:22,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:27,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:32,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:33,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:37:33,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:37:33,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:37:35,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:37:35,523 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:37:35,523 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:37:35,525 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:37:36,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:37:37,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:37:37,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:42,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:47,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:48,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:37:48,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:37:48,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:37:51,398 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:37:51,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:37:51,675 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:37:51,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:37:51,678 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:37:52,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:37:52,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:37:53,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:37:57,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:02,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:03,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:38:03,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:38:03,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:38:07,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:38:07,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:38:07,523 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:38:07,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:38:08,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:38:08,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:09,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:38:13,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:18,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:38:18,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:38:18,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:38:18,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:21,401 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:38:23,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:38:23,589 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:38:23,589 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:38:23,591 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:38:24,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:38:24,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:25,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:38:29,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:33,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:38:33,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:38:33,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:38:34,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:39,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:39,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:38:39,669 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:38:39,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:38:39,671 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:38:40,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:38:41,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:38:44,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:48,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:38:48,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:38:48,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:38:49,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:51,404 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:38:55,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:38:55,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:38:55,730 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:38:55,730 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:38:55,732 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:38:56,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:38:57,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:39:00,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:03,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:39:03,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:39:03,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:39:06,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:11,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:11,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:39:11,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:39:11,682 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:39:11,683 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:39:12,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:39:13,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:39:16,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:18,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:39:18,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:39:18,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:39:21,407 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:39:22,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:27,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:27,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:39:27,855 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:39:27,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:39:27,857 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:39:28,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:39:29,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:39:32,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:33,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:39:33,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:39:33,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:39:38,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:43,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:43,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:39:43,711 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:39:43,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:39:43,713 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:39:44,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:39:45,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:39:48,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:39:48,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:39:48,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:39:48,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:51,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:39:54,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:59,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:39:59,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:39:59,754 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:39:59,755 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:39:59,756 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:40:00,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:40:01,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:40:03,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:40:03,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: stop_status +2024-08-29 07:40:03,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: stop_status +2024-08-29 07:40:08,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:13,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:15,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:40:18,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:40:23,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:24,886 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:40:24,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:24,887 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:40:24,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:40:24,890 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:40:25,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:40:27,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:40:28,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:29,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:31,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:40:31,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:40:31,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:40:31,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:40:32,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:40:33,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:40:33,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:40:35,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:38,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:40,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:43,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:45,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:47,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:40:47,840 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:40:47,841 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:40:47,842 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:40:48,245 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:40:48,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:40:49,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:40:51,417 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:40:51,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:53,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:40:56,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:40:58,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:01,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:03,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:41:03,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:41:03,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:41:03,789 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:41:03,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:41:04,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:41:05,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:41:06,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:08,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:11,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:13,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:16,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:18,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:41:19,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:41:19,933 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:41:19,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:41:19,935 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:41:20,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:41:21,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:41:21,419 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:41:22,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:23,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:27,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:28,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:32,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:33,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:41:36,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:41:36,114 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:41:36,114 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:41:36,117 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:41:36,286 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:41:37,287 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:41:38,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:38,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:43,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:43,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:48,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:48,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:41:51,422 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:41:52,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:41:52,156 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:41:52,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:41:52,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:41:52,301 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:41:53,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:53,301 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:41:53,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:41:58,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:41:58,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:03,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:03,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:42:08,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:08,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:42:08,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:42:08,301 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:42:08,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:42:08,314 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:42:08,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:09,315 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:42:13,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:13,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:18,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:18,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:42:21,425 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:42:23,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:23,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:24,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:42:24,263 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:42:24,263 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:42:24,264 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:42:24,328 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:42:25,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:42:28,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:29,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:33,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:42:34,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:38,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:39,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:40,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:42:40,214 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:42:40,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:42:40,218 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:42:40,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:42:41,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:42:43,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:45,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:42:50,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:51,427 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:42:53,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:42:55,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:42:56,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:42:56,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:42:56,566 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:42:56,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:42:57,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:42:57,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:42:58,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:00,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:03,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:43:05,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:08,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:10,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:12,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:43:12,417 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:43:12,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:43:12,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:43:13,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:43:13,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:43:13,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:16,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:18,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:43:21,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:21,431 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:43:23,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:26,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:28,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:28,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:43:28,479 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:43:28,480 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:43:28,481 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:43:29,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:43:29,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:43:31,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:33,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:43:36,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:38,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:41,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:43,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:44,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:43:44,637 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:43:44,637 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:43:44,639 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:43:45,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:43:45,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:43:46,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:43:51,434 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:43:52,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:53,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:43:57,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:43:58,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:00,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:44:00,779 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:44:00,779 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:44:00,781 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:44:01,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:44:01,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:44:02,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:03,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:44:07,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:08,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:12,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:13,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:16,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:44:16,812 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:44:16,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:44:16,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:44:17,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:44:17,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:18,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:44:19,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:44:21,438 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:44:23,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:23,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:28,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:28,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:33,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:44:33,078 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:44:33,078 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:44:33,080 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:44:33,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:44:33,445 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:44:34,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:35,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:44:38,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:39,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:43,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:44,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:44:49,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:49,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:44:49,229 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:44:49,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:44:49,232 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:44:49,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:44:51,440 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:44:51,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:44:53,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:54,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:44:58,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:44:59,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:03,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:45:04,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:05,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:45:05,280 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:45:05,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:45:05,281 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:45:05,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:45:07,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:45:08,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:10,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:13,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:15,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:18,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:45:20,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:21,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:45:21,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:45:21,316 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:45:21,318 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:45:21,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:45:21,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:45:23,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:23,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:45:25,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:28,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:30,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:33,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:45:35,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:37,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:45:37,573 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:45:37,573 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:45:37,575 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:45:38,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:38,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:45:39,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:45:40,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:43,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:45,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:48,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:45:50,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:51,447 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:45:53,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:45:53,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:45:53,653 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:45:53,654 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:45:53,655 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:45:54,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:45:55,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:45:55,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:45:58,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:00,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:03,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:46:05,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:08,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:09,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:46:09,901 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:46:09,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:46:09,903 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:46:10,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:46:10,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:11,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:46:13,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:15,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:18,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:46:20,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:21,449 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:46:23,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:26,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:46:26,116 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:46:26,117 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:46:26,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:26,118 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:46:26,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:46:27,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:46:28,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:31,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:33,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:46:36,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:38,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:41,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:42,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:46:42,079 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:46:42,079 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:46:42,081 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:46:42,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:46:43,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:43,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:46:47,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:48,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:46:51,452 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:46:52,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:53,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:57,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:46:58,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:46:58,328 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:46:58,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:46:58,330 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:46:58,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:46:58,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:46:59,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:47:03,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:47:08,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:08,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:13,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:13,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:14,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:47:14,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:47:14,578 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:47:14,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:47:14,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:47:15,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:47:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:47:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:21,455 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:47:23,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:24,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:28,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:29,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:30,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:47:30,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:47:30,426 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:47:30,427 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:47:30,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:47:31,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:47:33,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:47:35,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:38,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:40,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:43,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:45,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:46,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:47:46,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:47:46,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:47:46,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:47:46,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:47:47,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:47:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:47:50,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:51,459 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:47:53,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:47:56,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:47:58,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:01,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:02,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:48:02,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:48:02,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:48:02,608 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:48:02,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:48:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:48:03,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:48:06,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:08,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:11,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:13,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:16,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:48:18,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:48:18,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:48:18,556 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:48:18,557 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:48:18,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:48:19,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:48:21,461 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:48:22,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:23,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:27,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:28,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:32,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:33,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:48:34,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:48:34,512 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:48:34,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:48:34,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:48:34,653 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:48:35,653 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:48:37,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:38,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:42,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:43,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:47,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:48:50,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:48:50,561 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:48:50,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:48:50,563 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:48:50,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:48:51,464 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:48:51,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:48:53,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:53,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:48:58,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:48:58,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:03,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:49:03,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:06,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:49:06,607 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:49:06,607 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:49:06,609 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:49:06,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:49:07,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:49:08,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:08,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:13,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:13,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:49:18,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:21,468 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:49:22,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:49:22,445 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:49:22,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:49:22,447 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:49:22,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:49:23,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:23,695 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:49:24,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:28,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:29,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:33,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:49:34,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:38,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:38,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:49:38,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:49:38,544 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:49:38,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:49:38,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:49:39,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:39,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:49:43,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:44,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:48,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:49:49,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:51,470 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:49:53,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:49:54,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:49:54,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:49:54,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:49:54,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:49:54,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:49:55,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:49:55,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:49:58,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:00,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:50:05,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:08,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:10,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:10,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:50:10,523 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:50:10,524 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:50:10,525 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:50:10,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:50:11,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:50:13,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:15,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:18,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:50:20,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:21,473 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:50:23,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:26,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:26,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:50:26,602 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:50:26,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:50:26,604 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:50:26,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:50:27,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:50:28,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:31,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:50:36,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:38,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:41,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:42,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:50:42,667 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:50:42,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:50:42,670 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:50:42,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:50:43,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:43,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:50:46,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:48,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:50:51,476 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:50:52,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:53,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:57,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:50:58,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:50:58,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:50:58,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:50:58,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:50:58,786 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:50:59,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:50:59,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:51:02,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:51:07,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:08,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:12,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:13,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:14,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:51:14,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:51:14,883 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:51:14,884 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:51:15,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:51:15,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:51:17,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:18,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:51:21,480 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:51:23,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:23,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:28,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:28,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:31,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:51:31,020 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:51:31,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:51:31,022 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:51:31,807 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:51:31,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:51:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:51:34,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:38,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:39,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:43,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:44,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:47,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:51:47,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:51:47,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:51:47,109 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:51:47,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:51:47,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:51:48,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:51:49,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:51,483 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:51:53,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:54,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:51:58,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:51:59,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:03,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:52:03,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:52:03,142 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:52:03,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:52:03,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:52:03,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:52:03,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:52:05,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:08,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:10,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:13,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:15,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:52:19,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:52:19,275 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:52:19,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:52:19,277 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:52:19,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:52:19,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:52:20,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:21,486 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:52:23,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:25,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:28,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:30,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:33,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:52:35,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:52:35,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:52:35,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:52:35,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:52:35,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:52:35,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:52:36,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:38,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:41,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:43,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:46,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:48,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:52:51,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:51,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:52:51,391 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:52:51,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:52:51,393 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:52:51,489 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:52:51,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:52:53,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:52:53,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:52:56,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:52:58,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:01,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:03,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:53:06,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:07,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:53:07,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:53:07,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:53:07,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:53:07,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:53:08,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:09,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:53:11,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:13,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:16,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:53:21,492 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:53:22,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:23,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:53:23,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:53:23,517 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:53:23,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:53:23,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:23,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:53:25,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:53:27,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:28,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:32,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:33,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:53:37,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:38,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:39,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:53:39,462 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:53:39,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:53:39,464 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:53:39,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:53:41,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:53:43,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:43,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:48,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:53:48,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:51,523 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:53:53,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:53:53,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:55,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:53:55,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:53:55,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:53:55,750 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:53:55,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:53:57,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:53:58,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:53:58,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:03,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:54:03,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:08,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:08,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:11,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:54:11,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:54:11,707 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:54:11,708 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:54:11,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:54:13,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:13,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:54:14,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:54:19,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:21,527 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:54:23,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:25,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:27,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:54:27,661 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:54:27,661 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:54:27,662 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:54:27,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:54:28,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:29,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:54:30,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:54:35,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:38,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:40,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:43,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:43,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:54:43,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:54:43,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:54:43,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:54:43,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:54:45,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:45,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:54:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:54:50,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:51,529 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:54:53,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:56,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:54:58,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:54:59,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:54:59,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:54:59,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:54:59,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:55:00,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:55:01,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:01,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:55:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:55:06,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:08,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:11,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:13,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:16,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:55:16,042 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:55:16,043 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:55:16,044 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:55:17,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:55:17,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:18,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:55:18,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:55:21,532 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:55:22,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:23,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:27,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:28,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:32,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:55:32,185 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:55:32,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:55:32,187 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:55:33,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:55:33,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:55:34,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:55:38,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:38,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:43,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:43,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:48,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:55:48,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:55:48,514 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:55:48,515 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:55:48,516 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:55:49,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:55:50,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:55:51,536 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:55:53,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:53,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:55:58,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:55:58,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:56:03,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:04,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:56:04,594 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:56:04,594 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:56:04,595 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:56:05,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:56:06,058 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:56:08,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:08,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:13,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:13,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:18,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:56:18,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:20,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:56:20,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:56:20,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:56:20,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:56:21,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:56:21,546 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:56:22,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:56:23,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:24,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:28,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:29,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:33,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:56:34,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:37,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:56:37,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:56:37,049 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:56:37,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:56:37,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:56:38,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:56:38,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:40,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:43,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:45,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:56:50,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:51,549 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:56:52,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:56:52,907 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:56:52,908 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:56:52,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:56:53,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:56:53,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:56:54,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:56:55,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:56:58,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:00,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:03,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:57:05,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:08,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:09,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:57:09,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:57:09,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:57:09,138 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:57:10,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:57:10,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:57:11,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:13,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:16,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:57:21,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:21,552 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:57:23,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:25,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:57:25,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:57:25,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:57:25,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:57:26,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:57:26,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:57:26,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:28,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:31,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:33,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:57:36,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:38,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:41,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:57:41,261 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:57:41,262 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:57:41,263 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:57:42,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:57:42,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:57:42,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:43,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:47,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:57:51,557 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:57:52,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:53,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:57:57,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:57:57,298 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:57:57,299 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:57:57,300 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:57:58,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:57:58,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:57:58,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:57:58,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:03,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:03,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:58:08,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:08,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:13,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:13,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:58:13,434 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:58:13,434 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:58:13,435 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:58:13,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:14,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:58:14,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:58:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:58:18,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:21,559 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:58:23,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:23,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:28,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:28,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:29,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:58:29,402 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:58:29,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:58:29,404 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:58:30,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:58:30,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:58:33,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:58:34,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:38,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:39,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:43,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:44,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:45,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:58:45,261 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:58:45,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:58:45,263 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:58:46,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:58:46,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:58:48,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:58:50,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:51,562 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:58:53,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:58:55,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:58:58,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:00,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:01,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:59:01,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:59:01,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:59:01,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:59:02,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:59:02,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:59:03,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:59:06,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:08,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:11,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:13,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:16,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:17,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:59:17,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:59:17,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:59:17,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:59:18,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:59:18,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:59:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:59:21,566 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:59:21,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:23,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:26,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:28,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:31,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:33,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:59:33,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:59:33,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:59:33,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:59:33,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:59:34,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:59:34,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:59:37,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:38,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:42,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:43,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:47,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 07:59:49,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 07:59:49,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 07:59:49,702 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 07:59:49,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 07:59:50,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 07:59:50,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 07:59:51,567 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 07:59:52,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:53,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 07:59:57,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 07:59:58,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:02,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:00:05,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:00:05,658 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:00:05,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:00:05,660 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:00:06,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:00:06,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:00:07,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:08,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:12,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:13,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:17,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:00:21,570 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:00:21,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:00:21,705 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:00:21,705 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:00:21,706 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:00:22,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:00:22,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:00:22,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:23,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:27,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:28,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:32,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:00:37,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:00:37,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:00:37,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:00:37,750 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:00:37,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:38,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:00:38,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:00:38,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:42,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:43,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:47,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:00:51,573 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:00:53,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:00:53,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:53,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:00:53,798 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:00:53,798 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:00:53,800 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:00:54,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:00:54,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:00:58,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:00:58,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:01:03,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:08,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:08,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:09,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:01:09,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:01:09,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:01:09,858 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:01:10,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:01:10,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:01:13,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:13,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:01:18,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:21,577 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:01:23,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:24,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:25,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:01:25,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:01:25,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:01:25,851 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:01:26,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:01:26,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:01:28,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:29,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:01:34,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:38,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:39,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:41,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:01:41,998 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:01:41,999 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:01:42,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:01:42,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:01:43,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:44,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:01:45,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:01:50,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:51,579 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:01:53,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:01:55,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:01:58,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:01:58,032 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:01:58,033 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:01:58,034 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:01:58,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:01:58,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:00,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:02:01,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:02:06,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:08,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:11,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:13,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:14,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:02:14,197 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:02:14,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:02:14,199 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:02:14,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:02:16,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:16,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:02:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:02:21,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:21,582 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:02:23,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:26,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:28,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:30,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:02:30,342 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:02:30,343 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:02:30,344 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:02:30,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:02:32,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:32,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:02:33,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:02:37,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:38,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:42,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:43,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:46,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:02:46,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:02:46,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:02:46,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:02:46,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:02:48,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:02:48,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:02:51,587 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:02:53,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:53,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:02:58,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:02:58,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:02,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:03:02,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:03:02,323 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:03:02,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:03:02,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:03:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:03:04,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:04,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:03:08,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:09,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:13,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:14,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:03:18,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:03:18,462 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:03:18,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:03:18,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:03:19,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:03:19,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:20,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:03:21,589 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:03:23,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:24,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:28,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:29,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:33,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:03:34,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:03:34,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:03:34,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:03:34,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:03:34,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:03:35,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:36,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:03:38,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:40,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:43,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:45,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:03:50,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:50,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:03:50,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:03:50,508 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:03:50,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:03:51,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:03:51,592 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:03:52,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:03:53,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:03:55,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:03:58,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:00,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:04:05,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:06,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:04:06,658 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:04:06,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:04:06,660 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:04:07,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:04:08,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:04:08,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:10,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:13,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:15,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:04:20,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:21,595 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:04:22,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:04:22,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:04:22,613 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:04:22,615 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:04:23,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:04:23,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:24,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:04:26,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:28,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:31,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:33,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:04:36,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:38,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:04:38,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:04:38,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:04:38,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:04:38,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:39,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:04:40,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:04:42,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:43,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:47,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:48,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:04:51,599 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:04:52,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:53,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:04:54,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:04:54,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:04:54,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:04:54,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:04:55,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:04:56,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:04:57,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:04:58,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:02,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:03,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:05:07,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:08,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:10,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:05:10,887 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:05:10,888 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:05:10,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:05:11,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:05:12,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:05:12,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:13,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:17,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:05:21,600 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:05:23,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:23,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:26,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:05:26,926 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:05:26,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:05:26,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:05:27,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:05:28,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:05:28,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:28,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:05:33,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:38,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:38,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:43,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:05:43,060 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:05:43,060 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:05:43,061 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:05:43,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:05:43,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:44,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:44,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:05:48,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:05:49,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:51,603 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:05:53,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:54,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:05:58,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:05:59,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:05:59,217 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:05:59,218 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:05:59,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:05:59,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:06:00,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:00,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:06:03,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:06:05,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:08,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:10,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:13,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:15,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:15,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:06:15,398 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:06:15,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:06:15,400 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:06:15,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:06:16,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:06:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:06:20,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:21,607 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:06:23,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:25,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:28,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:30,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:31,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:06:31,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:06:31,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:06:31,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:06:31,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:06:32,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:06:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:06:36,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:38,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:41,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:43,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:46,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:47,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:06:47,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:06:47,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:06:47,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:06:48,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:06:48,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:06:48,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:06:51,610 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:06:51,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:53,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:06:56,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:06:58,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:01,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:03,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:07:03,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:07:03,725 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:07:03,725 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:07:03,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:07:04,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:07:04,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:07:06,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:08,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:11,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:13,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:16,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:18,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:07:19,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:07:19,957 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:07:19,958 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:07:19,959 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:07:20,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:07:20,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:07:21,613 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:07:22,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:23,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:27,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:28,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:32,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:33,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:07:36,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:07:36,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:07:36,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:07:36,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:07:36,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:07:36,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:07:38,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:38,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:43,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:43,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:48,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:48,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:07:51,616 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:07:52,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:07:52,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:07:52,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:07:52,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:07:52,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:07:52,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:07:53,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:54,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:07:58,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:07:59,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:03,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:08:04,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:07,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:08:07,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:08:07,999 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:08:08,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:08:08,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:08:08,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:08:08,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:10,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:13,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:15,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:18,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:08:20,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:21,619 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:08:23,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:24,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:08:24,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:08:24,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:08:24,223 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:08:24,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:08:24,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:08:25,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:28,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:30,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:33,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:08:35,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:38,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:40,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:08:40,066 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:08:40,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:08:40,068 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:08:40,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:08:40,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:08:41,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:43,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:46,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:48,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:08:51,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:51,622 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:08:53,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:08:56,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:08:56,039 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:08:56,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:08:56,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:08:56,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:08:56,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:08:57,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:08:58,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:02,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:03,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:09:07,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:08,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:12,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:12,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:09:12,100 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:09:12,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:09:12,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:09:12,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:09:12,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:09:13,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:17,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:18,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:09:21,625 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:09:22,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:23,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:27,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:28,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:09:28,252 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:09:28,253 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:09:28,254 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:09:28,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:28,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:09:28,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:09:33,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:33,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:09:38,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:38,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:43,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:43,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:44,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:09:44,108 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:09:44,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:09:44,110 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:09:44,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:09:44,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:09:48,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:09:49,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:51,630 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:09:53,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:54,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:09:58,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:09:59,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:00,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:10:00,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:10:00,305 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:10:00,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:10:00,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:10:00,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:10:03,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:10:05,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:08,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:10,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:13,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:15,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:16,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:10:16,337 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:10:16,337 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:10:16,339 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:10:16,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:10:16,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:10:18,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:10:20,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:21,631 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:10:23,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:25,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:28,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:30,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:32,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:10:32,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:10:32,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:10:32,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:10:32,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:10:32,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:10:33,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:10:36,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:38,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:41,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:43,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:46,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:48,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:10:48,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:10:48,535 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:10:48,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:10:48,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:10:48,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:10:50,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:10:51,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:51,633 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:10:53,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:10:56,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:10:58,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:01,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:03,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:11:04,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:11:04,692 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:11:04,692 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:11:04,694 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:11:04,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:11:06,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:06,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:11:08,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:11,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:13,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:16,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:18,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:11:20,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:11:20,747 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:11:20,748 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:11:20,749 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:11:20,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:11:21,638 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:11:22,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:22,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:11:23,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:27,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:28,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:32,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:33,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:11:36,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:11:36,705 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:11:36,705 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:11:36,706 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:11:36,859 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:11:37,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:38,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:38,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:11:42,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:43,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:47,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:48,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:11:51,667 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:11:52,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:11:52,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:11:52,833 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:11:52,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:52,835 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:11:52,875 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:11:53,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:11:54,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:11:57,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:11:58,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:02,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:03,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:12:07,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:08,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:08,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:12:08,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:12:08,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:12:08,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:12:09,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:12:10,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:12:12,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:13,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:17,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:18,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:12:21,671 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:12:23,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:23,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:24,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:12:24,930 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:12:24,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:12:24,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:12:25,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:12:26,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:12:28,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:28,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:33,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:12:33,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:38,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:38,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:41,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:12:41,068 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:12:41,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:12:41,070 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:12:41,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:12:42,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:12:43,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:44,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:48,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:12:49,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:51,674 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:12:53,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:54,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:12:56,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:12:57,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:12:57,002 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:12:57,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:12:57,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:12:58,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:12:58,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:13:00,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:03,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:13:05,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:08,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:10,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:13,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:13:13,127 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:13:13,128 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:13:13,129 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:13:13,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:13,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:13:14,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:13:15,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:18,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:13:20,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:21,678 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:13:23,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:25,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:28,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:29,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:13:29,361 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:13:29,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:13:29,363 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:13:29,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:13:30,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:13:31,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:33,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:13:36,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:38,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:41,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:43,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:45,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:13:45,374 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:13:45,374 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:13:45,376 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:13:45,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:13:46,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:46,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:13:48,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:13:51,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:51,680 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:13:53,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:13:56,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:13:58,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:01,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:14:01,328 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:14:01,328 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:14:01,330 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:14:01,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:14:02,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:02,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:14:03,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:14:07,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:08,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:12,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:13,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:17,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:17,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:14:17,478 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:14:17,479 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:14:17,481 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:14:18,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:14:18,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:14:19,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:14:21,683 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:14:22,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:23,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:27,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:28,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:32,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:33,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:14:33,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:14:33,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:14:33,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:14:33,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:14:34,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:14:35,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:14:38,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:38,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:43,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:43,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:48,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:14:48,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:49,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:14:49,377 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:14:49,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:14:49,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:14:50,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:14:51,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:14:51,687 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:14:53,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:53,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:14:58,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:14:58,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:03,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:15:03,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:05,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:15:05,864 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:15:05,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:15:05,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:15:06,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:15:07,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:15:08,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:08,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:13,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:13,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:18,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:15:18,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:21,690 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:15:21,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:15:21,816 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:15:21,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:15:21,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:15:22,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:15:23,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:15:23,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:24,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:28,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:29,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:33,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:15:34,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:37,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:15:37,867 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:15:37,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:15:37,869 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:15:38,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:15:38,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:39,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:15:39,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:43,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:44,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:15:49,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:51,698 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:15:53,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:53,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:15:53,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:15:53,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:15:53,928 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:15:54,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:15:54,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:15:55,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:15:58,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:15:59,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:16:04,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:08,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:09,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:10,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:16:10,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:16:10,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:16:10,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:16:11,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:16:11,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:16:13,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:15,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:18,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:16:20,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:21,701 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:16:23,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:25,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:26,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:16:26,155 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:16:26,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:16:26,157 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:16:27,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:16:27,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:16:28,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:31,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:33,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:16:36,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:38,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:41,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:42,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:16:42,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:16:42,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:16:42,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:16:43,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:16:43,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:16:43,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:46,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:16:51,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:51,704 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:16:53,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:56,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:16:58,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:16:58,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:16:58,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:16:58,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:16:58,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:16:59,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:16:59,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:17:02,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:17:07,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:08,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:12,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:13,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:14,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:17:14,412 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:17:14,413 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:17:14,414 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:17:15,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:17:15,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:17:18,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:17:18,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:21,709 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:17:23,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:23,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:28,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:28,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:30,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:17:30,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:17:30,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:17:30,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:17:31,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:17:31,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:17:33,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:17:34,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:38,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:39,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:43,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:44,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:46,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:17:46,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:17:46,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:17:46,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:17:47,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:17:47,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:17:48,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:17:49,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:51,712 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:17:53,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:17:55,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:17:58,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:00,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:02,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:18:02,865 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:18:02,865 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:18:02,867 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:18:03,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:18:03,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:18:03,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:18:05,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:08,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:10,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:13,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:15,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:18,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:18:18,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:18:18,879 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:18:18,880 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:18:18,881 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:18:19,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:18:19,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:18:20,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:21,717 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:18:23,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:26,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:28,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:31,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:33,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:18:34,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:18:34,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:18:34,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:18:34,929 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:18:35,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:18:35,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:18:36,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:38,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:41,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:43,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:46,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:48,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:18:51,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:18:51,003 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:18:51,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:18:51,005 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:18:51,239 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:18:51,719 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:18:52,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:18:52,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:53,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:18:57,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:18:58,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:02,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:03,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:19:06,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:19:06,866 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:19:06,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:19:06,868 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:19:07,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:19:07,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:08,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:08,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:12,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:13,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:13,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:16,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:18,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:18,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:19:18,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:21,721 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:19:24,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:24,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:24,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:26,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:19:28,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:29,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:29,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:33,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:19:34,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:39,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:39,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:42,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:19:42,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:19:42,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:19:42,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:19:43,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:19:44,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:19:44,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:44,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:48,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:19:49,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:51,726 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:19:54,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:19:55,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:19:59,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:19:59,082 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:19:59,083 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:19:59,084 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:19:59,296 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:19:59,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:00,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:20:01,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:03,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:20:06,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:09,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:11,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:14,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:15,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:20:15,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:20:15,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:20:15,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:20:15,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:20:16,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:18,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:20:18,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:20:21,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:21,727 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:20:24,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:26,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:29,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:31,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:20:31,336 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:20:31,337 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:20:31,338 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:20:32,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:20:32,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:33,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:20:34,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:20:37,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:39,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:42,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:44,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:47,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:47,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:20:47,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:20:47,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:20:47,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:20:48,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:20:48,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:20:50,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:20:51,730 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:20:52,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:54,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:20:57,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:20:59,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:02,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:03,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:21:03,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:21:03,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:21:03,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:21:03,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:21:04,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:21:06,351 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:21:08,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:09,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:13,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:14,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:18,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:21:18,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:19,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:21:19,445 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:21:19,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:21:19,446 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:21:20,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:21:21,733 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:21:22,365 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:21:23,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:24,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:28,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:29,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:33,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:21:33,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:35,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:21:35,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:21:35,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:21:35,484 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:21:36,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:21:38,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:21:39,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:39,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:44,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:44,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:48,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:21:49,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:51,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:21:51,547 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:21:51,547 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:21:51,548 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:21:51,736 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:21:52,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:21:54,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:54,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:21:54,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:21:59,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:21:59,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:03,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:22:04,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:07,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:22:07,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:22:07,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:22:07,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:22:08,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:22:09,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:10,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:22:10,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:14,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:15,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:18,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:22:20,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:21,738 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:22:23,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:22:23,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:22:23,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:22:23,412 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:22:23,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:22:24,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:26,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:22:26,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:29,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:31,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:33,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:22:36,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:39,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:22:39,272 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:22:39,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:22:39,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:22:39,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:39,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:22:40,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:22:42,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:44,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:47,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:48,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:22:51,742 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:22:52,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:54,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:22:55,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:22:55,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:22:55,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:22:55,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:22:55,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:22:58,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:22:58,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:22:59,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:03,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:03,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:23:08,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:09,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:11,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:23:11,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:23:11,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:23:11,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:23:11,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:23:13,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:14,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:14,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:23:18,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:18,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:23:21,747 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:23:23,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:24,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:27,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:23:27,383 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:23:27,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:23:27,385 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:23:27,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:23:29,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:29,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:30,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:23:33,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:23:34,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:39,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:39,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:43,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:23:43,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:23:43,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:23:43,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:23:43,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:23:44,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:44,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:23:45,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:48,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:23:50,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:51,748 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:23:54,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:55,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:23:59,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:23:59,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:23:59,623 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:23:59,623 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:23:59,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:24:00,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:24:01,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:02,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:24:03,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:24:06,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:09,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:11,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:14,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:15,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:24:15,481 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:24:15,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:24:15,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:24:15,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:24:17,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:18,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:24:18,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:24:21,752 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:24:22,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:24,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:27,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:29,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:31,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:24:31,524 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:24:31,525 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:24:31,527 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:24:32,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:24:33,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:24:33,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:34,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:24:38,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:39,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:43,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:44,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:47,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:24:47,620 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:24:47,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:24:47,622 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:24:48,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:24:48,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:24:48,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:50,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:24:51,756 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:24:53,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:54,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:24:58,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:24:59,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:03,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:25:03,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:25:03,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:25:03,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:25:03,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:25:04,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:25:04,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:06,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:25:09,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:09,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:14,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:14,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:18,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:25:19,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:19,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:25:19,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:25:19,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:25:19,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:25:20,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:25:21,760 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:25:22,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:25:24,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:24,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:29,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:29,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:33,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:25:34,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:35,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:25:35,803 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:25:35,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:25:35,805 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:25:36,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:25:38,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:25:39,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:39,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:44,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:44,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:48,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:25:49,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:51,765 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:25:51,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:25:51,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:25:51,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:25:51,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:25:52,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:25:54,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:54,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:25:54,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:25:59,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:25:59,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:03,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:26:04,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:07,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:26:07,995 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:26:07,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:26:07,997 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:26:08,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:26:09,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:10,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:10,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:26:14,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:15,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:18,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:26:20,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:21,766 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:26:24,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:26:24,155 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:26:24,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:26:24,157 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:26:24,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:24,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:26:25,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:26,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:26:29,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:30,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:33,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:26:35,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:39,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:40,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:40,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:26:40,289 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:26:40,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:26:40,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:26:40,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:26:42,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:26:44,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:45,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:48,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:26:50,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:51,770 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:26:54,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:26:55,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:26:56,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:26:56,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:26:56,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:26:56,485 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:26:56,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:26:58,634 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:26:59,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:01,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:03,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:27:06,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:09,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:11,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:12,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:27:12,416 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:27:12,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:27:12,418 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:27:12,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:27:14,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:14,646 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:27:17,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:18,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:27:21,772 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:27:22,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:24,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:27,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:28,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:27:28,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:27:28,745 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:27:28,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:27:29,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:29,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:27:30,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:27:33,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:27:33,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:38,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:39,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:43,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:44,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:44,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:27:44,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:27:44,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:27:44,803 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:27:45,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:27:46,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:27:48,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:27:48,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:51,774 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:27:53,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:54,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:27:58,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:27:59,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:00,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:28:00,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:28:00,923 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:28:00,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:28:01,685 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:28:02,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:28:03,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:28:04,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:08,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:09,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:13,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:14,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:17,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:28:17,245 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:28:17,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:28:17,247 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:28:17,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:28:18,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:28:18,699 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:28:20,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:21,779 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:28:23,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:25,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:28,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:30,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:33,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:28:33,403 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:28:33,403 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:28:33,405 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:28:33,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:28:33,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:28:34,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:28:36,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:38,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:41,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:43,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:46,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:48,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:28:49,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:28:49,445 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:28:49,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:28:49,447 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:28:49,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:28:50,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:28:51,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:51,815 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:28:53,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:28:56,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:28:58,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:01,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:03,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:29:05,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:29:06,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:29:06,002 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:29:06,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:29:06,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:29:07,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:08,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:08,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:29:12,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:13,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:17,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:18,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:29:21,818 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:29:22,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:29:22,291 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:29:22,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:29:22,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:22,293 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:29:22,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:29:23,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:24,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:29:27,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:28,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:32,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:33,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:29:37,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:38,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:29:38,360 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:29:38,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:29:38,362 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:29:38,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:38,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:29:40,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:29:42,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:43,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:47,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:48,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:29:51,822 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:29:52,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:53,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:29:54,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:29:54,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:29:54,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:29:54,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:29:54,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:29:56,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:29:58,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:29:58,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:03,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:30:03,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:08,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:08,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:10,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:30:10,577 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:30:10,578 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:30:10,579 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:30:10,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:30:12,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:30:13,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:13,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:18,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:30:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:21,825 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:30:23,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:23,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:26,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:30:26,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:30:26,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:30:26,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:30:27,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:30:28,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:28,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:30:29,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:33,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:30:34,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:38,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:39,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:42,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:30:42,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:30:42,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:30:42,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:30:43,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:43,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:30:44,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:30:44,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:48,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:30:49,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:51,828 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:30:53,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:55,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:30:58,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:30:58,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:30:58,995 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:30:58,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:30:58,997 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:30:59,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:31:00,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:31:00,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:03,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:31:05,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:08,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:10,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:13,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:15,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:31:15,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:31:15,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:31:15,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:31:15,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:31:16,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:16,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:31:18,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:31:21,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:21,831 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:31:23,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:26,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:28,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:31,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:31:31,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:31:31,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:31:31,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:31:31,860 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:31:32,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:32,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:31:33,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:31:37,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:38,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:42,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:43,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:47,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:47,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:31:47,077 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:31:47,077 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:31:47,079 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:31:47,872 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:31:48,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:31:48,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:31:51,835 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:31:52,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:53,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:31:57,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:31:58,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:02,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:03,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:32:03,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:32:03,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:32:03,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:32:03,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:32:03,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:32:04,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:32:08,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:08,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:13,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:13,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:18,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:18,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:32:18,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:32:18,984 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:32:18,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:32:18,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:32:19,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:32:20,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:32:21,837 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:32:23,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:23,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:28,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:28,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:33,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:32:33,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:34,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:32:34,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:32:34,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:32:34,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:32:35,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:32:36,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:32:38,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:38,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:43,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:43,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:48,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:32:48,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:50,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:32:50,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:32:50,776 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:32:50,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:32:50,920 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:32:51,840 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:32:52,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:32:53,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:54,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:32:58,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:32:59,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:03,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:33:04,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:06,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:33:06,720 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:33:06,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:33:06,722 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:33:06,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:33:08,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:08,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:33:10,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:13,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:15,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:18,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:33:20,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:21,842 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:33:22,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:33:22,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:33:22,702 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:33:22,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:33:22,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:33:23,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:24,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:33:26,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:28,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:31,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:33,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:33:36,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:38,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:38,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:33:38,827 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:33:38,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:33:38,829 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:33:38,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:33:40,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:33:41,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:43,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:46,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:48,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:33:51,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:51,845 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:33:53,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:33:54,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:33:54,796 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:33:54,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:33:54,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:33:54,971 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:33:56,972 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:33:57,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:33:58,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:02,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:03,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:34:07,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:08,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:10,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:34:10,839 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:34:10,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:34:10,841 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:34:10,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:34:12,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:12,985 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:34:13,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:17,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:18,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:34:21,849 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:34:22,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:23,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:26,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:34:26,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:34:26,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:34:26,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:34:26,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:34:27,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:28,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:28,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:34:32,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:33,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:34:37,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:38,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:42,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:34:42,947 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:34:42,947 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:34:42,948 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:34:43,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:34:43,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:43,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:45,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:34:48,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:34:48,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:51,851 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:34:53,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:54,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:34:58,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:34:58,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:34:58,998 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:34:58,998 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:34:58,999 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:34:59,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:35:00,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:01,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:35:03,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:35:05,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:08,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:10,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:13,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:14,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:35:14,943 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:35:14,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:35:14,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:35:15,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:35:15,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:17,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:35:18,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:35:20,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:21,856 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:35:23,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:26,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:28,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:31,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:35:31,144 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:35:31,144 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:35:31,146 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:35:32,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:35:32,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:33,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:35:33,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:35:37,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:38,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:42,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:43,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:47,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:47,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:35:47,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:35:47,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:35:47,293 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:35:48,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:35:48,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:35:49,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:35:51,858 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:35:52,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:53,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:35:57,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:35:58,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:02,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:03,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:36:03,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:36:03,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:36:03,255 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:36:03,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:36:04,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:36:05,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:36:08,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:08,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:13,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:13,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:18,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:18,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:36:19,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:36:19,446 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:36:19,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:36:19,448 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:36:20,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:36:21,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:36:21,860 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:36:23,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:23,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:28,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:28,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:33,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:36:33,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:35,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:36:35,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:36:35,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:36:35,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:36:36,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:36:37,103 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:36:38,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:39,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:43,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:44,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:48,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:36:49,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:51,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:36:51,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:36:51,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:36:51,765 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:36:51,864 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:36:52,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:36:53,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:36:53,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:54,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:36:58,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:36:59,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:03,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:37:04,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:08,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:37:08,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:37:08,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:37:08,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:37:08,128 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:37:08,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:09,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:37:10,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:13,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:15,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:18,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:37:20,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:21,866 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:37:23,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:23,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:37:23,990 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:37:23,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:37:23,992 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:37:24,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:37:25,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:37:25,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:28,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:30,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:33,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:37:35,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:38,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:40,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:37:40,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:37:40,311 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:37:40,312 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:37:41,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:37:41,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:37:41,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:43,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:46,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:48,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:37:51,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:51,870 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:37:53,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:56,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:37:56,434 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:37:56,435 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:37:56,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:37:56,436 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:37:57,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:37:58,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:37:59,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:38:01,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:03,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:38:06,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:08,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:11,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:12,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:38:12,470 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:38:12,471 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:38:12,472 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:38:13,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:38:13,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:15,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:38:16,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:18,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:38:21,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:21,873 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:38:23,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:26,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:28,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:28,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:38:28,837 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:38:28,838 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:38:28,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:38:29,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:38:31,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:38:32,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:33,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:38:37,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:38,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:42,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:43,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:44,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:38:44,790 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:38:44,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:38:44,792 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:38:45,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:38:47,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:38:48,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:38:48,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:51,876 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:38:53,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:53,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:38:58,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:38:58,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:00,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:39:00,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:39:00,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:39:00,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:39:01,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:39:03,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:39:03,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:39:03,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:08,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:08,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:13,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:13,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:17,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:39:17,084 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:39:17,084 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:39:17,086 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:39:17,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:39:18,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:39:19,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:19,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:39:21,878 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:39:23,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:24,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:28,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:29,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:33,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:39:33,224 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:39:33,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:39:33,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:39:33,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:39:33,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:39:35,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:35,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:39:38,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:40,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:43,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:45,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:48,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:39:49,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:39:49,350 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:39:49,350 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:39:49,352 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:39:50,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:39:50,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:51,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:39:51,882 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:39:53,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:39:55,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:39:58,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:00,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:03,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:40:05,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:40:05,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:40:05,523 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:40:05,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:40:06,271 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:40:06,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:07,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:40:08,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:11,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:13,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:16,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:18,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:40:21,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:40:21,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:40:21,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:40:21,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:40:21,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:21,886 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:40:22,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:40:23,286 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:40:23,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:26,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:28,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:31,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:33,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:40:36,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:37,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:40:37,623 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:40:37,623 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:40:37,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:40:38,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:40:38,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:39,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:40:42,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:43,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:47,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:48,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:40:51,887 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:40:52,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:53,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:40:53,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:40:53,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:40:53,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:40:53,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:40:54,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:40:55,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:40:58,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:40:58,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:03,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:41:03,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:08,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:08,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:09,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:41:09,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:41:09,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:41:09,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:41:10,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:41:11,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:41:13,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:14,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:18,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:41:19,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:21,891 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:41:23,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:24,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:25,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:41:25,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:41:25,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:41:25,552 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:41:26,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:41:27,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:41:28,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:30,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:33,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:41:35,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:38,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:40,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:41,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:41:41,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:41:41,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:41:41,612 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:41:42,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:41:43,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:41:43,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:45,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:48,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:41:50,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:51,894 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:41:53,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:55,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:41:57,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:41:57,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:41:57,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:41:57,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:41:58,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:41:58,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:41:59,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:42:01,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:03,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:42:06,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:08,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:11,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:13,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:13,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:42:13,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:42:13,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:42:13,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:42:14,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:42:15,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:42:16,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:18,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:42:21,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:21,897 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:42:23,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:26,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:28,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:29,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:42:29,663 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:42:29,663 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:42:29,664 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:42:30,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:42:31,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:42:32,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:33,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:42:37,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:38,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:42,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:43,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:45,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:42:45,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:42:45,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:42:45,725 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:42:46,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:42:47,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:42:47,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:48,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:42:51,900 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:42:52,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:53,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:42:57,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:42:58,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:01,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:43:01,566 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:43:01,566 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:43:01,568 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:43:02,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:43:03,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:43:03,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:43:03,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:08,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:08,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:13,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:13,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:17,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:43:17,512 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:43:17,513 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:43:17,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:43:18,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:43:18,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:43:19,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:43:19,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:21,903 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:43:23,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:24,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:28,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:29,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:33,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:43:33,470 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:43:33,471 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:43:33,472 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:43:33,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:43:34,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:43:35,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:43:35,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:38,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:40,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:43,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:45,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:48,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:43:49,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:43:49,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:43:49,522 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:43:49,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:43:50,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:43:50,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:51,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:43:51,907 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:43:53,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:43:55,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:43:58,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:00,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:03,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:44:05,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:44:05,482 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:44:05,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:44:05,484 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:44:06,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:44:06,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:07,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:44:08,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:11,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:13,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:16,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:18,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:44:21,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:21,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:44:21,640 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:44:21,641 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:44:21,642 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:44:21,909 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:44:22,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:44:23,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:44:23,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:26,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:28,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:31,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:33,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:44:36,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:37,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:44:37,673 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:44:37,673 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:44:37,675 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:44:38,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:44:38,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:39,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:44:42,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:43,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:47,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:48,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:44:51,911 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:44:52,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:53,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:44:53,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:44:53,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:44:53,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:44:53,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:44:54,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:44:55,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:44:58,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:44:58,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:03,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:45:03,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:08,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:08,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:09,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:45:09,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:45:09,675 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:45:09,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:45:10,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:45:11,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:45:13,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:13,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:18,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:45:18,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:21,915 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:45:23,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:23,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:25,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:45:25,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:45:25,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:45:25,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:45:26,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:45:27,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:45:28,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:29,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:33,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:45:34,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:38,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:39,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:41,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:45:41,783 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:45:41,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:45:41,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:45:42,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:45:43,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:45:43,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:44,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:48,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:45:49,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:51,950 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:45:53,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:54,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:45:57,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:45:57,923 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:45:57,923 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:45:57,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:45:58,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:45:58,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:45:59,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:46:00,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:03,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:46:05,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:08,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:10,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:13,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:13,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:46:13,880 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:46:13,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:46:13,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:46:14,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:46:15,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:46:16,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:18,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:46:21,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:21,953 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:46:23,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:26,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:28,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:29,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:46:29,893 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:46:29,894 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:46:29,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:46:30,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:46:31,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:46:32,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:33,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:46:37,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:38,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:42,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:43,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:46,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:46:46,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:46:46,180 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:46:46,182 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:46:46,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:46:47,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:46:48,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:48,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:46:51,956 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:46:53,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:53,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:46:58,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:46:58,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:02,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:47:02,311 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:47:02,312 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:47:02,314 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:47:02,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:47:03,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:47:03,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:47:04,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:08,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:09,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:13,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:14,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:18,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:47:18,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:47:18,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:47:18,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:47:18,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:47:18,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:47:19,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:19,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:47:21,959 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:47:23,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:24,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:28,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:29,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:33,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:47:34,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:47:34,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:47:34,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:47:34,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:47:34,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:47:35,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:35,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:47:38,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:40,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:43,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:45,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:48,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:47:50,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:50,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:47:50,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:47:50,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:47:50,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:47:51,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:47:51,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:47:51,962 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:47:53,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:47:55,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:47:58,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:00,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:03,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:48:05,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:06,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:48:06,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:48:06,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:48:06,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:48:07,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:48:07,661 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:48:08,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:11,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:13,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:16,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:18,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:48:21,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:21,965 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:48:23,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:48:23,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:48:23,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:48:23,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:48:23,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:48:23,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:25,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:48:27,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:28,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:32,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:33,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:48:37,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:38,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:39,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:48:39,282 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:48:39,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:48:39,284 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:48:39,687 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:48:41,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:48:42,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:43,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:47,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:48,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:48:51,970 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:48:52,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:53,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:48:55,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:48:55,336 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:48:55,336 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:48:55,338 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:48:55,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:48:57,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:48:58,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:48:58,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:03,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:03,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:49:08,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:08,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:11,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:49:11,393 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:49:11,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:49:11,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:49:11,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:49:13,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:13,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:49:13,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:18,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:18,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:49:21,972 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:49:23,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:23,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:27,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:49:27,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:49:27,605 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:49:27,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:49:27,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:49:28,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:29,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:29,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:49:33,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:49:34,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:38,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:39,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:43,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:49:43,659 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:49:43,659 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:49:43,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:49:43,739 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:49:43,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:44,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:45,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:49:48,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:49:49,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:51,975 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:49:53,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:54,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:49:58,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:49:59,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:49:59,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:49:59,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:49:59,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:50:00,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:50:00,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:01,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:50:03,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:50:05,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:08,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:10,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:13,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:15,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:15,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:50:15,981 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:50:15,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:50:15,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:50:16,765 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:50:17,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:50:18,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:50:20,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:21,977 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:50:23,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:26,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:28,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:31,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:32,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:50:32,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:50:32,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:50:32,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:50:32,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:50:33,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:50:33,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:50:37,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:38,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:42,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:43,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:47,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:48,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:50:48,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:50:48,142 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:50:48,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:50:48,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:50:48,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:50:49,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:50:51,980 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:50:52,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:53,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:50:57,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:50:58,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:02,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:03,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:51:04,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:51:04,194 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:51:04,194 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:51:04,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:51:04,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:51:05,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:51:08,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:08,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:13,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:13,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:18,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:18,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:51:20,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:51:20,363 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:51:20,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:51:20,365 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:51:20,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:51:21,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:51:21,983 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:51:23,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:23,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:28,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:28,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:33,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:51:33,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:36,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:51:36,307 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:51:36,308 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:51:36,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:51:36,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:51:37,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:51:38,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:39,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:43,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:44,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:48,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:51:49,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:51,988 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:51:52,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:51:52,343 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:51:52,343 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:51:52,345 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:51:52,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:51:53,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:51:53,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:54,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:51:58,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:51:59,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:03,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:52:04,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:08,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:52:08,498 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:52:08,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:52:08,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:52:08,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:52:08,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:09,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:09,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:52:13,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:14,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:18,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:52:19,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:21,992 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:52:23,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:24,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:52:24,582 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:52:24,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:52:24,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:24,584 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:52:24,874 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:52:25,874 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:52:28,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:29,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:33,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:52:34,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:38,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:39,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:40,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:52:40,430 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:52:40,431 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:52:40,432 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:52:40,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:52:41,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:52:43,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:45,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:48,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:52:50,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:51,994 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:52:53,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:52:55,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:52:56,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:52:56,384 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:52:56,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:52:56,385 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:52:56,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:52:57,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:52:58,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:01,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:03,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:53:06,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:08,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:11,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:12,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:53:12,323 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:53:12,324 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:53:12,325 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:53:12,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:53:13,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:53:13,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:17,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:18,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:53:21,997 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:53:22,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:23,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:27,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:28,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:53:28,279 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:53:28,280 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:53:28,282 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:53:28,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:53:28,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:29,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:53:33,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:33,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:53:38,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:38,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:43,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:43,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:44,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:53:44,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:53:44,236 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:53:44,237 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:53:44,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:53:45,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:53:48,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:53:49,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:52,002 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:53:53,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:53:55,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:53:58,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:00,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:00,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:54:00,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:54:00,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:54:00,294 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:54:00,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:54:01,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:54:03,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:54:05,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:08,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:10,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:13,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:15,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:16,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:54:16,355 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:54:16,356 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:54:16,357 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:54:16,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:54:17,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:54:18,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:54:20,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:22,003 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:54:23,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:26,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:28,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:31,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:32,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:54:32,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:54:32,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:54:32,401 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:54:32,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:54:33,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:54:33,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:54:36,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:38,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:41,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:43,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:46,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:48,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:54:48,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:54:48,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:54:48,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:54:48,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:54:48,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:54:49,998 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:54:51,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:52,007 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:54:53,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:54:57,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:54:58,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:02,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:03,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:55:04,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:55:04,523 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:55:04,524 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:55:04,525 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:55:05,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:55:06,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:55:07,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:08,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:12,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:13,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:17,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:18,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:55:20,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:55:20,580 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:55:20,580 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:55:20,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:55:21,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:55:22,007 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:55:22,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:55:23,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:23,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:28,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:28,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:33,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:33,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:55:36,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:55:36,535 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:55:36,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:55:36,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:55:37,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:55:38,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:55:38,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:38,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:43,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:43,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:48,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:55:48,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:52,012 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:55:52,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:55:52,624 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:55:52,624 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:55:52,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:55:53,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:55:53,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:53,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:55:54,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:55:58,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:55:58,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:03,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:56:03,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:08,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:08,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:56:08,953 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:56:08,953 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:56:08,954 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:56:08,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:09,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:56:10,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:56:13,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:13,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:18,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:56:18,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:22,014 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:56:23,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:24,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:24,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:56:24,809 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:56:24,810 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:56:24,812 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:56:25,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:56:26,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:56:28,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:29,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:33,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:56:34,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:38,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:39,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:40,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:56:40,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:56:40,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:56:40,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:56:41,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:56:42,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:56:43,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:44,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:48,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:56:49,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:52,017 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:56:53,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:56:55,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:56:57,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:56:57,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:56:57,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:56:57,227 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:56:58,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:56:58,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:56:58,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:00,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:03,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:57:05,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:08,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:10,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:13,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:57:13,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:57:13,348 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:57:13,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:57:13,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:14,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:57:14,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:57:15,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:18,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:57:20,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:22,020 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:57:23,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:26,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:28,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:29,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:57:29,484 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:57:29,485 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:57:29,487 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:57:30,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:57:30,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:57:31,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:33,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:57:36,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:38,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:41,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:43,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:45,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:57:45,539 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:57:45,539 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:57:45,540 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:57:46,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:57:46,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:57:46,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:48,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:57:51,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:52,023 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:57:53,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:57:57,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:57:58,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:01,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:58:01,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:58:01,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:58:01,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:58:02,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:58:02,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:03,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:58:04,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:58:07,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:08,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:12,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:13,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:17,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:17,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:58:17,973 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:58:17,973 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:58:17,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:58:18,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:58:18,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:58:20,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:58:22,027 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:58:23,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:23,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:28,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:28,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:33,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:33,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:58:34,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:58:34,098 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:58:34,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:58:34,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:58:34,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:58:36,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:58:38,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:38,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:43,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:43,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:48,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:48,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:58:50,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:58:50,368 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:58:50,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:58:50,370 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:58:51,191 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:58:52,030 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:58:52,191 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:58:53,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:54,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:58:58,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:58:59,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:03,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:59:04,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:06,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:59:06,307 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:59:06,307 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:59:06,308 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:59:07,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:59:08,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:59:08,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:09,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:13,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:14,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:18,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:59:19,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:22,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:59:22,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:59:22,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:59:22,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:59:22,382 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:59:23,216 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:59:23,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:24,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:59:24,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:28,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:29,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:33,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:59:34,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:38,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:59:38,430 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:59:38,430 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:59:38,432 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:59:38,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:39,228 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:59:39,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:40,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:59:43,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:44,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:48,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 08:59:49,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:52,038 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 08:59:53,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:54,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 08:59:54,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 08:59:54,581 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 08:59:54,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 08:59:54,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 08:59:55,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 08:59:56,242 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 08:59:58,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 08:59:59,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:03,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:00:04,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:08,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:09,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:10,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:00:10,514 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:00:10,514 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:00:10,516 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:00:11,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:00:12,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:00:13,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:15,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:18,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:00:20,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:22,041 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:00:23,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:26,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:26,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:00:26,737 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:00:26,738 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:00:26,739 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:00:27,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:00:28,267 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:00:28,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:31,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:33,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:00:36,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:38,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:41,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:42,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:00:42,909 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:00:42,909 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:00:42,911 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:00:43,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:00:43,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:44,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:00:46,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:48,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:00:51,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:52,045 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:00:54,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:57,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:00:58,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:00:58,977 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:00:58,977 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:00:58,979 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:00:59,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:00:59,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:01:00,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:01:02,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:03,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:01:07,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:09,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:12,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:14,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:14,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:01:14,930 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:01:14,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:01:14,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:01:15,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:01:16,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:01:18,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:01:18,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:22,047 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:01:24,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:24,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:29,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:29,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:30,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:01:30,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:01:30,955 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:01:30,956 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:01:31,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:01:32,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:01:33,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:01:34,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:39,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:39,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:44,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:44,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:47,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:01:47,088 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:01:47,088 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:01:47,089 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:01:47,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:01:48,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:01:48,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:01:50,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:52,049 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:01:54,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:01:56,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:01:59,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:01,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:03,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:02:03,024 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:02:03,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:02:03,026 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:02:03,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:02:03,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:02:04,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:02:07,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:09,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:12,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:14,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:17,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:18,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:02:19,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:02:19,060 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:02:19,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:02:19,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:02:19,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:02:20,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:02:22,052 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:02:22,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:24,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:27,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:29,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:32,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:33,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:02:35,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:02:35,226 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:02:35,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:02:35,228 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:02:35,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:02:36,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:02:37,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:39,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:42,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:44,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:47,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:48,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:02:51,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:02:51,161 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:02:51,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:02:51,163 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:02:51,382 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:02:52,057 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:02:52,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:02:53,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:54,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:02:58,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:02:59,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:03,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:03,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:03:07,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:03:07,011 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:03:07,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:03:07,013 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:03:07,394 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:03:08,395 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:03:09,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:09,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:14,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:14,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:18,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:03:19,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:22,058 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:03:23,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:03:23,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:03:23,090 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:03:23,092 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:03:23,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:03:24,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:24,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:24,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:03:29,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:29,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:33,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:03:34,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:39,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:39,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:03:39,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:39,137 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:03:39,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:03:39,139 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:03:39,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:03:40,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:03:44,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:44,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:48,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:03:49,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:52,064 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:03:54,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:03:54,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:03:55,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:03:55,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:03:55,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:03:55,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:03:55,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:03:56,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:03:59,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:00,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:03,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:04:05,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:09,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:10,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:11,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:04:11,174 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:04:11,174 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:04:11,175 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:04:11,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:04:12,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:04:14,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:15,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:18,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:04:20,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:22,068 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:04:24,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:26,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:27,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:04:27,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:04:27,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:04:27,290 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:04:27,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:04:28,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:04:29,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:31,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:33,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:04:36,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:39,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:41,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:43,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:04:43,509 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:04:43,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:04:43,511 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:04:44,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:44,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:04:44,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:04:46,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:48,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:04:51,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:52,070 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:04:54,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:57,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:04:59,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:04:59,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:04:59,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:04:59,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:04:59,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:04:59,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:05:00,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:05:02,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:03,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:05:07,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:09,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:12,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:14,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:15,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:05:15,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:05:15,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:05:15,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:05:16,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:05:16,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:05:17,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:18,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:05:22,073 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:05:23,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:24,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:28,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:29,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:31,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:05:31,660 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:05:31,661 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:05:31,663 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:05:32,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:05:32,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:05:33,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:05:33,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:38,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:39,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:43,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:44,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:47,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:05:47,715 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:05:47,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:05:47,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:05:48,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:05:48,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:05:48,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:05:48,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:52,076 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:05:54,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:54,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:05:59,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:05:59,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:03,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:06:03,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:06:03,955 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:06:03,955 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:06:03,957 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:06:04,538 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:06:04,538 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:06:04,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:09,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:09,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:14,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:14,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:18,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:06:20,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:20,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:06:20,178 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:06:20,178 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:06:20,179 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:06:20,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:06:22,079 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:06:22,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:06:24,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:25,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:29,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:30,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:33,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:06:35,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:36,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:06:36,198 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:06:36,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:06:36,200 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:06:36,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:06:38,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:06:39,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:40,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:44,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:45,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:48,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:06:50,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:52,082 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:06:52,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:06:52,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:06:52,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:06:52,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:06:52,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:06:54,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:06:54,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:06:55,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:06:59,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:00,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:03,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:07:05,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:08,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:07:08,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:07:08,556 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:07:08,557 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:07:08,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:07:09,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:10,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:07:10,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:14,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:15,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:18,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:07:20,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:22,084 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:07:24,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:24,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:07:24,611 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:07:24,612 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:07:24,614 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:07:25,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:25,615 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:07:26,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:07:29,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:30,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:33,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:07:35,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:39,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:40,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:40,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:07:40,661 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:07:40,662 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:07:40,663 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:07:41,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:07:42,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:07:44,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:45,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:48,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:07:50,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:52,089 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:07:54,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:07:56,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:07:56,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:07:56,734 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:07:56,734 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:07:56,735 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:07:57,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:07:58,640 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:07:59,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:01,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:03,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:08:06,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:09,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:11,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:12,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:08:12,957 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:08:12,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:08:12,959 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:08:13,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:08:14,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:14,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:08:16,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:18,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:08:21,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:22,091 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:08:24,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:27,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:28,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:08:28,908 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:08:28,909 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:08:28,910 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:08:29,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:29,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:08:30,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:08:32,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:33,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:08:37,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:39,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:42,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:44,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:45,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:08:45,072 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:08:45,072 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:08:45,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:08:45,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:08:46,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:08:48,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:48,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:08:52,094 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:08:53,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:54,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:08:58,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:08:59,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:01,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:09:01,319 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:09:01,319 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:09:01,321 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:09:01,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:09:02,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:09:03,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:03,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:09:08,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:09,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:13,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:14,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:17,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:09:17,493 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:09:17,493 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:09:17,495 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:09:17,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:09:18,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:18,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:09:18,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:09:22,099 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:09:24,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:24,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:29,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:29,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:33,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:09:33,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:09:33,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:09:33,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:09:33,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:09:33,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:09:34,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:34,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:09:39,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:39,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:44,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:44,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:48,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:09:49,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:49,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:09:49,771 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:09:49,771 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:09:49,773 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:09:50,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:09:50,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:09:52,100 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:09:54,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:09:55,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:09:59,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:00,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:03,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:10:05,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:05,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:10:05,915 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:10:05,916 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:10:05,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:10:06,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:10:06,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:10:09,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:10,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:14,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:15,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:18,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:10:20,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:22,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:10:22,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:10:22,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:10:22,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:10:22,103 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:10:22,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:10:22,760 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:10:24,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:26,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:29,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:31,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:33,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:10:36,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:38,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:10:38,365 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:10:38,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:10:38,367 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:10:38,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:10:38,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:10:39,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:41,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:44,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:46,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:48,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:10:51,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:52,107 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:10:54,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:10:54,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:10:54,428 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:10:54,428 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:10:54,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:10:54,787 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:10:54,787 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:10:56,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:10:59,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:01,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:03,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:11:06,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:09,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:10,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:11:10,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:11:10,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:11:10,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:11:10,800 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:11:11,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:12,801 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:11:14,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:16,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:18,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:11:21,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:22,111 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:11:24,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:26,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:11:26,728 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:11:26,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:11:26,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:26,730 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:11:26,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:11:28,813 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:11:29,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:31,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:33,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:11:36,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:39,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:41,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:42,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:11:42,761 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:11:42,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:11:42,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:11:42,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:11:44,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:44,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:11:46,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:48,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:11:51,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:52,113 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:11:54,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:57,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:11:58,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:11:58,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:11:58,990 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:11:58,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:11:59,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:11:59,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:12:00,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:12:02,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:03,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:12:07,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:09,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:12,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:14,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:15,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:12:15,011 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:12:15,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:12:15,013 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:12:15,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:12:16,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:12:18,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:18,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:12:22,116 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:12:23,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:24,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:28,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:29,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:31,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:12:31,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:12:31,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:12:31,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:12:31,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:12:32,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:12:33,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:12:34,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:39,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:39,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:44,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:44,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:47,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:12:47,188 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:12:47,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:12:47,190 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:12:47,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:12:48,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:12:48,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:12:49,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:52,121 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:12:54,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:12:55,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:12:59,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:00,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:03,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:13:03,249 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:13:03,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:13:03,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:13:03,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:13:03,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:13:04,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:05,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:08,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:09,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:10,899 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:10,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:13,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:14,902 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:14,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:16,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:18,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:13:18,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:20,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:21,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:22,122 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:13:22,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:13:24,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:24,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:26,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:29,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:31,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:33,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:13:36,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:38,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:13:38,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:13:38,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:13:38,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:13:38,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:13:39,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:40,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:41,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:44,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:46,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:48,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:13:51,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:52,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:13:54,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:13:55,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:13:55,539 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:13:55,539 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:13:55,540 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:13:55,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:13:56,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:13:57,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:13:59,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:02,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:03,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:14:07,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:09,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:11,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:14:11,491 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:14:11,492 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:14:11,493 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:14:11,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:14:12,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:14:13,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:14,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:18,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:18,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:14:22,129 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:14:24,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:24,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:27,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:14:27,778 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:14:27,778 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:14:27,779 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:14:27,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:14:28,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:14:29,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:29,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:33,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:14:34,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:39,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:39,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:44,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:44,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:14:44,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:14:44,052 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:14:44,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:14:44,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:14:44,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:14:45,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:48,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:14:50,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:52,132 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:14:54,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:14:55,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:14:59,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:00,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:15:00,120 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:15:00,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:15:00,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:15:00,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:15:00,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:15:01,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:03,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:15:06,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:09,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:11,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:14,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:16,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:16,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:15:16,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:15:16,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:15:16,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:15:17,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:15:17,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:15:18,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:15:21,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:22,135 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:15:24,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:27,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:29,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:32,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:32,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:15:32,369 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:15:32,370 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:15:32,371 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:15:33,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:15:33,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:15:35,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:15:37,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:39,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:42,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:44,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:47,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:48,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:15:48,214 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:15:48,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:15:48,216 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:15:48,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:15:49,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:15:49,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:15:52,138 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:15:53,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:54,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:15:58,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:15:59,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:03,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:03,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:16:04,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:16:04,474 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:16:04,474 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:16:04,476 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:16:05,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:16:07,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:16:08,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:09,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:13,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:14,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:18,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:16:18,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:20,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:16:20,358 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:16:20,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:16:20,360 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:16:21,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:16:22,143 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:16:23,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:16:24,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:24,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:29,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:29,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:33,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:16:34,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:36,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:16:36,321 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:16:36,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:16:36,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:16:37,072 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:16:37,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:16:39,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:39,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:44,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:44,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:48,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:16:49,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:52,145 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:16:52,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:16:52,483 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:16:52,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:16:52,485 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:16:53,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:16:54,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:54,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:16:55,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:16:59,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:16:59,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:03,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:17:04,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:08,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:17:08,388 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:17:08,389 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:17:08,390 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:17:09,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:09,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:17:10,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:11,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:17:14,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:15,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:18,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:17:20,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:22,148 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:17:24,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:24,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:17:24,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:17:24,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:17:24,442 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:17:25,114 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:17:25,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:27,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:17:29,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:30,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:33,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:17:35,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:39,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:40,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:40,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:17:40,488 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:17:40,488 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:17:40,489 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:17:41,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:17:43,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:17:44,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:45,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:48,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:17:50,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:52,152 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:17:54,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:56,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:17:56,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:17:56,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:17:56,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:17:56,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:17:57,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:17:59,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:17:59,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:18:01,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:03,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:18:06,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:09,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:11,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:12,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:18:12,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:18:12,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:18:12,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:18:13,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:18:13,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:18:14,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:17,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:18,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:18:22,154 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:18:23,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:24,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:28,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:28,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:18:28,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:18:28,508 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:18:28,509 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:18:29,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:29,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:18:29,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:18:33,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:33,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:18:38,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:39,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:43,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:44,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:44,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:18:44,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:18:44,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:18:44,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:18:45,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:18:47,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:18:48,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:18:48,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:52,156 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:18:54,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:54,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:18:59,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:18:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:00,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:19:00,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:19:00,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:19:00,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:19:01,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:19:03,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:19:03,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:19:04,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:09,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:09,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:14,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:14,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:16,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:19:16,638 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:19:16,639 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:19:16,640 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:19:17,203 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:19:18,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:19:19,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:19:20,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:22,160 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:19:24,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:26,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:29,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:31,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:32,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:19:32,879 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:19:32,879 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:19:32,881 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:19:33,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:19:33,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:19:35,219 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:19:36,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:39,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:41,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:44,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:46,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:48,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:19:49,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:19:49,013 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:19:49,013 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:19:49,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:19:49,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:19:51,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:19:52,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:52,163 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:19:54,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:19:57,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:19:59,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:02,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:03,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:20:04,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:20:04,959 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:20:04,959 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:20:04,961 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:20:05,245 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:20:07,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:20:08,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:09,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:13,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:14,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:18,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:18,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:20:21,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:20:21,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:20:21,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:20:21,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:20:21,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:20:22,198 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:20:23,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:23,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:20:24,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:28,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:29,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:33,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:33,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:20:37,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:20:37,259 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:20:37,259 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:20:37,260 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:20:37,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:20:38,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:39,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:39,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:20:43,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:44,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:48,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:48,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:20:52,202 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:20:53,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:20:53,117 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:20:53,117 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:20:53,119 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:20:53,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:20:54,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:54,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:20:55,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:20:59,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:20:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:03,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:21:04,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:09,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:09,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:09,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:21:09,478 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:21:09,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:21:09,480 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:21:10,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:21:11,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:21:14,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:14,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:18,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:21:19,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:22,206 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:21:24,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:25,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:25,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:21:25,517 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:21:25,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:21:25,519 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:21:26,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:21:27,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:21:29,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:30,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:33,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:21:35,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:39,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:40,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:41,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:21:41,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:21:41,436 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:21:41,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:21:42,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:21:43,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:21:44,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:46,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:48,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:21:51,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:52,208 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:21:54,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:57,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:21:57,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:21:57,667 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:21:57,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:21:57,669 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:21:58,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:21:59,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:21:59,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:22:02,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:03,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:22:07,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:09,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:12,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:13,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:22:13,822 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:22:13,823 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:22:13,824 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:22:14,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:14,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:22:15,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:22:17,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:18,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:22:22,211 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:22:23,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:24,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:28,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:29,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:30,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:22:30,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:22:30,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:22:30,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:22:30,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:22:31,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:22:33,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:22:34,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:39,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:39,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:44,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:44,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:46,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:22:46,209 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:22:46,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:22:46,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:22:46,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:22:47,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:22:48,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:22:49,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:52,214 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:22:54,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:54,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:22:59,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:22:59,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:02,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:23:02,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:23:02,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:23:02,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:23:03,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:23:03,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:23:03,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:23:04,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:09,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:09,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:14,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:14,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:18,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:23:18,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:23:18,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:23:18,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:23:18,667 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:23:19,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:23:19,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:23:19,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:22,217 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:23:24,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:25,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:29,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:30,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:33,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:23:34,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:23:34,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:23:34,997 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:23:34,999 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:23:35,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:23:36,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:37,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:23:39,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:41,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:44,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:46,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:48,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:23:51,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:23:51,018 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:23:51,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:23:51,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:23:51,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:51,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:23:52,220 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:23:53,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:23:54,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:23:56,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:23:59,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:01,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:03,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:24:06,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:07,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:24:07,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:24:07,536 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:24:07,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:24:08,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:24:09,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:09,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:24:11,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:14,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:16,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:18,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:24:21,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:22,222 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:24:23,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:24:23,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:24:23,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:24:23,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:24:24,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:24,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:24:25,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:24:26,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:29,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:31,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:33,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:24:36,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:39,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:39,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:24:39,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:24:39,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:24:39,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:24:40,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:24:41,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:24:41,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:44,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:46,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:48,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:24:51,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:52,227 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:24:54,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:24:55,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:24:55,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:24:55,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:24:55,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:24:56,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:24:56,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:24:57,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:24:59,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:01,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:03,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:25:06,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:09,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:11,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:25:11,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:25:11,777 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:25:11,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:25:12,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:25:12,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:13,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:25:14,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:17,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:18,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:25:22,229 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:25:23,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:24,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:27,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:25:27,806 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:25:27,806 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:25:27,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:25:28,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:25:28,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:29,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:29,504 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:25:33,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:25:33,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:38,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:39,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:43,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:43,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:25:43,860 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:25:43,860 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:25:43,861 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:25:44,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:44,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:25:45,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:25:48,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:25:48,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:52,232 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:25:54,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:54,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:59,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:25:59,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:25:59,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:25:59,713 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:25:59,713 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:25:59,715 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:26:00,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:26:01,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:26:03,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:26:04,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:09,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:09,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:14,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:14,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:15,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:26:15,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:26:15,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:26:15,977 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:26:16,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:26:17,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:26:18,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:26:19,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:22,237 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:26:24,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:25,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:29,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:30,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:31,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:26:31,948 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:26:31,949 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:26:31,950 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:26:32,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:26:33,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:26:33,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:26:35,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:39,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:40,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:44,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:45,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:47,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:26:47,907 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:26:47,908 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:26:47,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:26:48,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:26:48,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:26:49,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:26:51,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:52,239 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:26:54,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:26:57,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:26:59,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:02,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:03,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:27:03,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:27:03,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:27:03,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:27:03,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:27:04,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:27:05,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:27:08,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:09,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:13,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:14,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:18,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:18,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:27:19,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:27:19,951 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:27:19,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:27:19,952 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:27:20,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:27:21,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:27:22,243 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:27:23,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:24,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:28,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:29,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:33,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:33,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:27:36,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:27:36,271 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:27:36,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:27:36,272 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:27:36,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:27:37,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:27:38,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:39,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:43,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:44,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:48,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:48,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:27:52,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:27:52,219 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:27:52,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:27:52,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:27:52,249 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:27:52,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:27:53,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:27:54,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:54,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:27:59,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:27:59,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:03,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:28:04,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:08,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:28:08,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:28:08,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:28:08,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:28:08,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:28:09,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:09,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:09,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:28:14,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:14,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:18,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:28:19,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:22,251 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:28:24,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:24,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:28:24,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:28:24,296 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:28:24,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:28:24,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:28:25,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:25,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:28:29,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:30,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:33,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:28:35,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:39,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:40,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:40,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:28:40,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:28:40,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:28:40,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:28:40,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:28:41,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:28:44,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:45,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:48,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:28:50,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:52,252 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:28:54,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:28:56,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:28:56,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:28:56,415 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:28:56,415 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:28:56,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:28:56,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:28:57,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:28:59,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:01,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:03,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:29:06,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:09,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:11,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:12,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:29:12,668 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:29:12,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:29:12,670 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:29:12,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:29:13,685 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:29:14,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:16,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:18,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:29:21,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:22,256 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:29:24,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:27,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:28,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:29:28,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:29:28,625 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:29:28,628 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:29:28,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:29:29,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:29,699 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:29:32,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:33,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:29:37,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:39,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:42,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:44,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:44,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:29:44,775 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:29:44,775 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:29:44,777 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:29:45,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:29:45,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:29:47,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:48,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:29:52,260 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:29:53,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:54,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:29:58,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:29:59,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:00,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:30:00,916 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:30:00,916 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:30:00,918 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:30:01,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:30:01,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:30:03,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:30:03,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:08,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:09,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:13,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:14,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:16,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:30:16,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:30:16,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:30:16,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:30:17,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:30:17,738 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:30:18,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:30:19,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:22,261 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:30:24,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:25,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:29,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:30,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:33,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:30:33,350 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:30:33,351 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:30:33,352 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:30:33,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:30:33,752 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:30:35,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:35,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:30:39,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:40,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:44,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:45,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:48,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:30:49,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:30:49,281 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:30:49,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:30:49,283 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:30:49,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:30:49,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:30:51,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:52,264 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:30:54,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:30:57,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:30:59,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:02,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:03,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:31:05,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:31:05,351 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:31:05,352 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:31:05,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:31:05,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:31:07,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:07,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:31:09,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:12,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:14,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:18,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:31:21,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:31:21,538 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:31:21,538 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:31:21,540 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:31:21,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:31:22,267 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:31:23,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:23,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:31:24,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:28,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:29,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:33,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:33,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:31:37,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:31:37,676 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:31:37,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:31:37,678 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:31:37,803 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:31:38,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:39,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:39,804 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:31:43,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:44,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:48,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:31:48,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:52,270 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:31:53,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:31:53,713 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:31:53,713 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:31:53,715 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:31:53,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:53,816 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:31:54,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:31:55,817 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:31:58,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:31:59,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:03,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:32:03,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:08,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:09,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:10,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:32:10,078 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:32:10,079 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:32:10,080 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:32:10,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:32:11,830 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:32:14,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:14,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:18,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:32:19,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:22,273 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:32:24,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:24,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:26,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:32:26,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:32:26,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:32:26,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:32:26,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:32:27,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:32:29,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:30,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:33,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:32:35,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:39,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:40,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:42,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:32:42,504 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:32:42,505 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:32:42,506 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:32:42,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:32:43,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:32:44,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:45,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:48,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:32:50,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:52,277 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:32:54,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:56,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:32:58,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:32:58,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:32:58,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:32:58,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:32:58,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:32:59,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:32:59,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:33:01,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:03,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:33:06,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:09,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:11,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:14,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:14,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:33:14,719 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:33:14,719 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:33:14,721 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:33:14,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:33:16,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:16,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:33:18,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:33:21,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:22,281 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:33:24,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:27,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:29,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:30,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:33:30,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:33:30,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:33:30,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:33:30,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:33:32,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:32,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:33:33,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:33:37,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:39,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:42,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:44,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:46,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:33:46,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:33:46,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:33:46,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:33:47,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:33:47,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:48,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:33:48,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:33:52,284 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:33:53,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:54,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:33:58,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:33:59,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:02,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:34:02,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:34:02,850 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:34:02,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:34:02,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:34:03,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:34:03,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:04,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:34:08,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:09,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:13,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:14,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:18,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:34:18,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:19,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:34:19,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:34:19,179 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:34:19,180 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:34:19,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:34:20,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:34:22,287 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:34:24,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:24,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:29,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:29,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:33,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:34:34,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:35,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:34:35,621 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:34:35,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:34:35,623 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:34:35,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:34:38,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:34:39,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:39,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:44,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:44,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:48,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:34:49,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:51,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:34:51,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:34:51,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:34:51,644 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:34:51,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:34:52,293 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:34:54,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:34:54,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:34:55,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:34:59,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:00,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:03,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:35:05,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:07,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:35:07,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:35:07,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:35:07,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:35:07,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:35:09,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:10,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:35:11,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:14,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:16,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:18,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:35:21,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:22,295 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:35:24,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:35:24,027 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:35:24,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:35:24,029 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:35:24,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:24,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:35:26,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:26,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:35:29,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:31,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:33,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:35:36,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:39,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:40,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:35:40,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:35:40,050 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:35:40,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:35:41,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:35:42,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:43,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:35:44,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:47,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:48,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:35:52,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:52,297 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:35:54,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:35:55,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:35:55,979 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:35:55,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:35:55,981 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:35:56,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:35:57,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:35:59,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:35:59,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:02,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:03,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:36:07,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:09,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:12,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:36:12,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:36:12,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:36:12,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:36:13,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:36:13,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:14,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:15,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:36:18,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:18,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:36:22,301 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:36:23,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:24,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:28,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:36:28,079 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:36:28,079 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:36:28,081 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:36:29,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:36:29,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:29,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:31,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:36:33,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:36:34,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:39,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:39,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:44,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:44,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:36:44,130 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:36:44,131 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:36:44,132 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:36:44,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:45,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:36:47,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:36:48,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:36:49,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:52,305 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:36:54,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:54,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:36:59,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:36:59,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:00,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:37:00,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:37:00,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:37:00,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:37:01,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:37:03,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:37:03,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:37:05,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:09,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:10,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:14,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:15,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:16,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:37:16,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:37:16,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:37:16,258 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:37:17,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:37:18,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:37:19,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:37:20,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:22,338 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:37:24,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:25,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:29,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:30,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:32,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:37:32,114 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:37:32,114 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:37:32,116 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:37:33,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:37:33,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:37:35,093 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:37:36,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:39,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:41,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:44,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:46,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:48,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:37:48,477 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:37:48,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:37:48,479 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:37:48,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:37:49,107 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:37:51,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:37:51,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:52,341 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:37:54,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:37:57,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:37:59,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:02,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:03,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:38:04,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:38:04,441 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:38:04,441 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:38:04,442 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:38:05,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:38:07,122 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:38:07,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:09,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:12,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:14,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:17,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:18,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:38:20,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:38:20,597 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:38:20,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:38:20,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:38:21,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:38:22,347 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:38:23,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:38:23,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:24,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:28,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:29,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:33,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:33,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:38:36,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:38:36,555 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:38:36,555 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:38:36,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:38:37,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:38:38,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:39,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:38:39,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:43,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:44,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:48,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:38:48,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:52,348 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:38:52,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:38:52,796 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:38:52,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:38:52,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:38:53,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:38:53,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:54,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:38:55,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:38:58,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:38:59,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:03,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:39:03,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:08,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:08,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:39:08,933 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:39:08,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:39:08,935 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:39:09,171 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:39:09,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:11,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:39:13,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:14,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:18,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:39:18,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:22,351 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:39:24,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:24,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:25,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:39:25,077 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:39:25,077 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:39:25,079 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:39:25,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:39:27,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:39:29,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:30,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:33,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:39:35,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:39,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:40,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:41,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:39:41,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:39:41,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:39:41,148 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:39:41,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:39:43,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:39:44,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:45,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:48,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:39:50,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:52,354 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:39:54,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:39:55,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:39:57,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:39:57,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:39:57,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:39:57,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:39:58,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:39:59,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:39:59,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:01,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:03,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:40:06,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:09,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:11,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:13,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:40:13,413 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:40:13,414 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:40:13,415 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:40:14,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:40:14,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:15,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:40:16,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:18,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:40:21,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:22,357 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:40:24,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:27,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:29,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:40:29,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:40:29,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:40:29,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:40:29,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:30,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:40:31,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:40:32,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:33,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:40:37,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:39,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:42,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:44,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:45,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:40:45,708 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:40:45,709 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:40:45,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:40:46,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:40:47,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:40:47,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:48,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:40:52,360 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:40:53,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:54,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:40:58,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:40:59,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:01,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:41:01,727 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:41:01,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:41:01,729 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:41:02,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:41:03,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:41:03,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:41:03,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:08,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:09,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:13,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:14,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:17,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:41:17,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:41:17,684 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:41:17,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:41:18,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:41:18,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:41:19,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:41:19,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:22,362 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:41:24,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:25,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:29,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:30,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:33,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:41:33,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:41:33,936 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:41:33,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:41:33,938 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:41:34,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:41:35,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:41:35,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:39,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:40,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:44,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:45,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:48,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:41:50,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:41:50,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:41:50,082 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:41:50,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:41:50,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:41:51,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:52,367 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:41:53,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:41:54,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:41:56,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:41:59,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:01,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:03,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:42:06,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:42:06,028 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:42:06,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:42:06,030 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:42:06,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:42:07,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:07,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:42:09,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:12,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:14,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:17,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:18,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:42:22,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:22,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:42:22,168 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:42:22,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:42:22,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:42:22,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:42:22,368 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:42:24,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:25,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:42:27,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:29,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:32,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:33,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:42:37,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:38,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:42:38,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:42:38,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:42:38,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:42:38,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:42:39,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:41,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:42:43,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:44,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:48,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:48,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:42:52,372 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:42:53,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:54,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:42:54,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:42:54,484 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:42:54,484 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:42:54,485 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:42:55,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:42:57,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:42:58,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:42:59,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:03,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:03,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:43:08,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:09,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:10,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:43:10,495 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:43:10,496 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:43:10,497 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:43:11,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:43:13,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:43:13,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:14,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:18,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:18,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:43:22,375 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:43:24,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:24,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:26,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:43:26,786 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:43:26,786 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:43:26,788 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:43:27,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:43:29,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:43:29,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:29,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:33,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:43:34,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:39,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:39,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:42,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:43:42,842 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:43:42,842 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:43:42,844 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:43:43,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:43:44,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:44,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:45,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:43:48,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:43:49,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:52,378 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:43:54,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:43:55,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:43:59,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:43:59,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:43:59,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:43:59,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:43:59,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:43:59,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:01,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:01,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:44:03,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:44:06,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:09,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:11,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:14,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:14,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:44:14,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:44:14,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:44:14,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:44:15,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:44:16,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:17,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:44:18,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:44:21,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:22,381 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:44:24,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:27,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:29,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:31,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:44:31,367 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:44:31,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:44:31,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:44:31,445 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:44:33,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:33,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:44:33,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:44:38,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:39,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:43,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:44,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:47,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:44:47,413 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:44:47,414 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:44:47,415 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:44:47,462 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:44:48,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:48,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:44:49,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:44:52,385 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:44:54,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:54,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:44:59,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:44:59,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:03,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:45:03,437 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:45:03,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:45:03,439 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:45:03,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:45:03,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:45:04,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:05,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:45:09,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:09,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:14,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:14,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:18,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:45:19,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:19,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:45:19,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:45:19,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:45:19,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:45:20,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:45:21,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:45:22,387 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:45:24,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:25,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:29,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:30,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:33,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:45:35,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:35,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:45:35,819 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:45:35,820 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:45:35,821 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:45:36,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:45:37,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:45:39,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:40,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:44,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:45,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:48,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:45:50,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:51,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:45:51,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:45:51,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:45:51,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:45:52,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:45:52,521 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:45:53,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:45:54,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:45:56,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:45:59,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:01,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:03,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:46:06,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:07,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:46:07,866 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:46:07,867 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:46:07,868 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:46:08,534 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:46:09,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:09,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:46:11,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:14,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:16,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:18,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:46:21,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:22,395 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:46:23,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:23,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:46:23,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:46:23,925 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:46:23,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:46:24,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:46:25,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:46:26,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:28,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:31,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:33,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:46:36,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:38,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:40,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:46:40,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:46:40,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:46:40,063 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:46:40,561 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:46:41,562 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:46:42,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:43,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:47,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:48,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:46:52,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:52,399 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:46:53,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:46:55,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:46:55,910 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:46:55,910 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:46:55,912 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:46:56,575 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:46:57,575 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:46:57,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:46:58,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:02,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:03,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:47:07,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:08,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:12,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:47:12,080 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:47:12,080 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:47:12,081 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:47:12,588 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:47:13,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:13,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:13,589 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:47:18,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:18,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:47:22,401 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:47:23,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:23,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:28,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:47:28,129 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:47:28,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:47:28,130 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:47:28,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:28,602 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:47:29,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:29,603 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:47:33,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:47:34,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:38,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:39,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:43,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:44,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:47:44,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:47:44,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:47:44,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:47:44,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:47:45,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:45,617 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:47:48,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:47:50,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:52,404 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:47:53,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:47:55,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:47:58,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:00,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:48:00,240 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:48:00,241 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:48:00,242 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:48:00,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:48:01,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:01,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:48:03,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:48:06,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:08,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:11,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:13,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:16,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:16,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:48:16,393 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:48:16,393 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:48:16,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:48:16,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:48:17,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:48:18,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:48:21,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:22,408 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:48:23,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:26,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:28,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:31,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:32,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:48:32,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:48:32,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:48:32,634 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:48:32,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:48:33,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:48:35,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:48:36,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:38,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:41,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:43,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:46,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:48,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:48:48,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:48:48,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:48:48,693 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:48:48,694 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:48:49,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:48:51,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:48:51,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:52,411 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:48:53,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:48:57,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:48:58,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:02,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:03,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:49:04,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:49:04,771 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:49:04,771 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:49:04,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:49:05,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:49:07,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:49:07,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:08,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:12,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:13,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:17,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:18,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:49:20,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:49:20,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:49:20,997 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:49:20,999 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:49:21,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:49:22,413 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:49:23,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:23,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:23,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:49:28,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:28,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:33,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:33,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:49:37,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:49:37,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:49:37,236 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:49:37,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:49:37,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:49:38,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:39,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:39,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:49:43,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:44,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:48,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:49:49,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:52,417 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:49:53,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:49:53,376 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:49:53,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:49:53,378 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:49:53,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:53,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:49:54,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:49:55,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:49:58,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:49:59,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:03,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:50:04,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:08,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:09,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:09,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:50:09,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:50:09,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:50:09,599 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:50:09,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:50:11,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:50:13,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:14,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:18,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:50:19,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:22,421 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:50:23,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:25,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:25,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:50:25,737 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:50:25,737 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:50:25,739 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:50:25,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:50:27,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:50:28,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:30,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:33,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:50:35,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:38,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:40,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:41,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:50:41,678 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:50:41,678 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:50:41,680 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:50:41,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:50:43,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:43,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:50:46,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:48,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:50:51,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:52,423 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:50:53,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:57,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:50:58,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:50:58,002 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:50:58,003 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:50:58,004 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:50:58,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:50:58,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:50:59,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:51:03,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:03,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:51:08,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:08,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:13,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:13,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:14,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:51:14,019 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:51:14,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:51:14,021 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:51:14,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:51:15,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:51:18,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:18,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:51:22,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:51:23,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:23,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:28,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:28,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:29,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:51:29,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:51:29,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:51:29,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:51:30,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:51:31,813 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:51:33,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:51:33,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:38,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:38,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:43,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:43,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:46,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:51:46,099 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:51:46,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:51:46,101 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:51:46,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:51:47,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:51:48,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:51:49,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:52,438 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:51:53,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:54,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:51:58,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:51:59,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:02,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:52:02,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:52:02,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:52:02,159 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:52:02,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:52:03,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:52:03,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:52:05,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:08,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:10,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:13,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:15,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:18,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:52:18,184 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:52:18,185 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:52:18,186 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:52:18,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:52:18,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:52:19,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:52:20,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:22,440 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:52:23,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:25,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:28,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:30,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:33,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:52:34,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:52:34,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:52:34,349 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:52:34,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:52:34,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:52:35,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:52:36,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:38,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:41,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:43,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:46,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:48,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:52:50,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:52:50,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:52:50,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:52:50,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:52:50,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:52:51,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:51,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:52:52,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:52:53,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:52:57,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:52:58,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:02,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:03,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:53:06,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:53:06,557 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:53:06,557 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:53:06,559 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:53:06,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:53:07,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:07,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:53:08,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:12,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:13,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:17,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:18,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:53:22,445 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:53:22,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:53:22,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:53:22,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:53:22,587 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:53:22,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:22,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:53:23,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:23,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:53:27,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:28,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:32,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:33,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:53:37,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:38,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:38,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:53:38,764 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:53:38,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:53:38,766 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:53:38,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:53:39,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:53:42,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:43,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:47,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:48,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:53:52,450 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:53:53,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:53:53,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:54,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:53:54,979 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:53:54,979 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:53:54,981 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:53:55,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:53:57,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:53:58,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:53:58,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:03,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:54:03,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:08,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:08,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:11,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:54:11,109 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:54:11,110 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:54:11,111 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:54:11,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:54:13,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:13,958 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:54:14,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:18,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:54:19,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:22,453 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:54:23,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:24,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:26,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:54:26,971 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:54:26,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:54:26,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:54:26,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:54:28,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:29,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:29,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:54:33,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:54:34,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:38,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:39,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:43,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:54:43,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:54:43,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:54:43,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:54:43,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:43,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:54:45,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:45,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:54:48,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:54:50,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:52,455 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:54:53,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:55,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:54:58,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:54:59,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:54:59,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:54:59,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:54:59,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:55:00,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:55:01,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:02,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:55:03,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:55:06,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:08,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:11,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:13,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:15,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:55:15,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:55:15,353 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:55:15,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:55:16,016 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:55:16,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:18,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:55:18,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:55:21,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:22,459 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:55:23,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:26,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:28,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:31,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:55:31,428 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:55:31,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:55:31,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:55:32,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:55:32,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:33,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:55:34,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:55:37,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:38,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:42,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:43,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:47,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:47,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:55:47,491 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:55:47,491 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:55:47,493 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:55:48,044 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:55:48,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:55:50,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:55:52,461 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:55:52,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:53,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:55:57,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:55:58,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:02,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:03,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:56:03,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:56:03,704 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:56:03,704 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:56:03,706 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:56:04,058 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:56:06,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:56:07,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:08,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:12,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:13,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:17,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:18,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:56:19,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:56:19,652 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:56:19,652 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:56:19,653 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:56:20,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:56:22,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:56:22,465 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:56:23,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:23,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:28,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:28,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:33,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:33,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:56:35,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:56:35,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:56:35,811 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:56:35,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:56:36,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:56:38,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:56:38,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:38,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:43,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:43,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:48,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:56:48,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:51,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:56:51,874 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:56:51,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:56:51,876 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:56:52,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:56:52,467 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:56:53,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:54,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:56:54,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:56:58,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:56:59,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:03,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:57:04,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:07,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:57:07,823 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:57:07,823 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:57:07,824 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:57:08,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:57:08,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:09,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:10,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:57:13,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:14,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:18,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:57:19,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:22,471 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:57:23,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:23,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:57:23,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:57:23,891 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:57:23,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:57:24,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:57:24,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:26,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:57:28,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:29,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:33,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:57:34,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:38,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:39,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:40,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:57:40,065 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:57:40,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:57:40,067 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:57:40,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:57:42,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:57:43,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:45,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:48,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:57:50,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:52,474 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:57:53,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:57:55,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:57:56,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:57:56,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:57:56,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:57:56,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:57:56,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:57:58,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:57:58,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:01,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:03,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:58:06,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:08,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:11,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:12,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:58:12,181 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:58:12,182 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:58:12,183 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:58:13,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:58:13,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:14,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:58:16,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:18,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:58:21,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:22,476 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:58:23,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:26,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:28,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:58:28,334 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:58:28,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:58:28,336 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:58:28,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:29,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:58:30,182 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:58:32,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:33,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:58:37,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:38,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:42,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:43,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:44,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:58:44,567 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:58:44,567 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:58:44,569 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:58:45,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:58:46,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:58:47,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:48,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:58:52,488 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:58:53,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:53,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:58:58,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:58:58,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:00,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:59:00,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:59:00,626 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:59:00,627 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:59:01,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:59:02,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:59:03,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:59:03,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:08,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:08,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:13,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:13,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:16,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:59:16,657 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:59:16,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:59:16,659 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:59:17,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:59:18,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:59:18,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:59:18,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:22,484 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:59:23,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:24,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:28,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:29,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:32,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:59:32,965 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:59:32,965 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:59:32,968 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:59:33,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:59:33,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:59:34,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:59:34,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:38,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:39,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:43,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:44,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:48,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 09:59:49,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 09:59:49,110 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 09:59:49,110 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 09:59:49,112 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 09:59:49,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 09:59:50,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:50,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 09:59:52,486 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 09:59:53,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 09:59:55,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 09:59:58,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:00,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:03,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:00:05,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:00:05,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:00:05,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:00:05,170 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:00:05,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:00:06,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:06,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:00:08,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:11,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:13,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:16,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:18,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:00:21,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:21,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:00:21,402 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:00:21,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:00:21,404 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:00:22,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:00:22,490 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:00:23,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:24,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:00:26,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:28,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:31,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:33,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:00:36,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:37,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:00:37,464 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:00:37,465 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:00:37,467 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:00:38,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:00:38,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:40,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:00:41,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:43,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:46,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:48,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:00:51,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:52,493 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:00:53,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:00:53,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:00:53,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:00:53,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:00:53,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:00:54,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:00:56,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:00:57,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:00:58,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:02,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:03,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:01:07,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:08,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:09,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:01:09,646 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:01:09,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:01:09,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:01:10,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:01:12,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:01:12,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:13,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:17,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:18,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:01:22,496 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:01:23,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:23,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:25,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:01:25,687 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:01:25,688 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:01:25,689 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:01:26,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:01:28,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:01:28,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:28,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:33,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:01:33,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:38,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:38,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:41,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:01:41,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:01:41,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:01:41,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:01:42,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:01:43,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:44,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:01:44,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:48,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:01:49,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:52,499 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:01:53,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:01:55,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:01:57,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:01:57,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:01:57,762 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:01:57,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:01:58,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:01:58,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:00,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:02:00,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:03,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:02:05,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:08,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:10,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:13,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:02:13,930 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:02:13,930 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:02:13,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:02:13,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:14,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:02:15,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:16,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:02:18,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:02:20,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:22,503 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:02:23,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:26,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:28,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:29,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:02:29,996 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:02:29,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:02:29,998 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:02:30,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:02:31,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:32,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:02:33,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:02:36,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:38,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:42,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:43,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:46,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:02:46,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:02:46,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:02:46,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:02:46,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:02:47,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:48,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:02:48,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:02:52,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:52,505 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:02:53,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:02:57,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:02:58,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:02,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:03:02,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:03:02,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:03:02,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:03:02,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:03:03,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:03,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:03:04,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:03:08,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:08,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:13,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:14,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:18,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:18,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:03:18,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:03:18,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:03:18,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:03:18,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:03:18,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:03:20,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:03:22,508 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:03:23,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:24,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:28,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:29,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:33,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:33,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:03:34,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:03:34,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:03:34,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:03:34,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:03:34,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:03:36,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:03:39,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:39,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:44,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:44,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:48,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:03:49,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:50,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:03:50,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:03:50,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:03:50,595 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:03:51,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:03:52,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:03:52,512 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:03:54,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:54,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:03:59,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:03:59,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:03,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:04:04,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:06,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:04:06,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:04:06,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:04:06,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:04:07,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:04:08,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:04:09,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:09,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:14,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:14,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:18,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:04:19,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:22,515 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:04:22,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:04:22,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:04:22,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:04:22,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:04:23,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:04:24,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:24,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:04:24,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:29,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:29,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:33,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:04:34,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:38,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:04:38,717 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:04:38,717 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:04:38,718 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:04:39,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:39,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:04:40,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:04:40,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:44,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:45,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:48,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:04:50,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:52,518 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:04:54,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:04:55,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:04:55,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:04:55,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:04:55,063 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:04:55,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:04:56,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:04:56,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:04:59,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:01,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:03,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:05:06,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:09,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:10,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:05:10,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:05:10,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:05:10,924 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:05:11,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:05:11,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:12,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:05:14,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:16,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:18,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:05:21,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:22,521 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:05:24,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:27,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:05:27,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:05:27,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:05:27,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:27,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:05:27,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:05:28,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:05:29,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:32,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:33,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:05:37,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:39,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:42,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:43,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:05:43,147 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:05:43,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:05:43,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:05:43,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:05:44,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:44,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:05:48,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:48,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:05:52,524 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:05:53,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:54,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:58,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:05:59,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:05:59,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:05:59,358 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:05:59,359 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:05:59,361 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:05:59,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:06:00,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:06:03,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:06:04,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:09,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:09,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:14,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:14,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:15,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:06:15,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:06:15,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:06:15,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:06:16,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:06:16,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:06:18,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:06:19,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:22,527 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:06:24,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:25,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:29,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:30,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:31,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:06:31,663 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:06:31,664 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:06:31,665 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:06:32,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:06:32,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:06:33,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:06:35,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:39,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:40,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:44,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:45,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:47,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:06:47,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:06:47,631 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:06:47,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:06:48,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:06:48,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:06:48,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:06:51,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:52,530 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:06:54,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:06:57,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:06:59,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:02,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:03,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:07:03,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:07:03,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:07:03,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:07:03,870 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:07:04,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:07:06,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:08,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:08,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:09,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:12,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:13,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:13,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:14,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:14,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:18,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:07:19,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:20,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:22,536 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:07:22,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:23,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:07:24,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:24,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:25,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:29,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:30,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:33,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:07:35,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:39,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:07:39,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:07:39,348 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:07:39,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:07:39,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:39,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:07:40,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:40,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:44,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:45,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:48,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:07:50,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:52,537 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:07:54,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:07:55,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:07:55,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:07:55,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:07:55,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:07:55,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:07:56,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:07:56,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:07:59,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:00,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:03,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:08:05,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:09,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:10,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:11,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:08:11,765 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:08:11,766 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:08:11,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:08:12,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:08:12,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:08:14,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:15,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:18,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:08:20,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:22,540 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:08:24,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:26,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:27,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:08:27,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:08:27,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:08:27,989 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:08:28,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:08:29,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:30,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:08:31,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:33,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:08:36,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:39,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:41,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:44,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:08:44,107 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:08:44,108 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:08:44,109 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:08:44,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:44,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:08:46,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:08:47,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:48,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:08:52,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:52,545 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:08:54,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:08:57,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:08:59,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:00,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:09:00,068 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:09:00,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:09:00,070 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:09:00,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:09:02,751 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:09:03,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:03,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:09:08,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:09,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:13,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:14,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:16,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:09:16,368 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:09:16,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:09:16,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:09:16,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:09:18,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:18,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:09:18,764 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:09:22,547 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:09:23,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:24,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:28,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:29,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:32,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:09:32,423 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:09:32,424 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:09:32,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:09:32,780 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:09:33,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:09:34,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:34,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:09:39,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:39,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:44,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:44,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:48,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:09:48,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:09:48,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:09:48,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:09:48,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:09:48,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:09:50,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:50,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:09:52,549 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:09:54,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:09:55,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:09:59,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:00,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:03,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:10:04,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:10:04,360 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:10:04,360 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:10:04,362 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:10:04,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:10:06,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:06,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:10:09,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:11,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:14,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:16,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:18,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:10:20,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:10:20,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:10:20,420 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:10:20,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:10:20,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:10:21,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:22,552 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:10:22,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:10:24,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:26,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:29,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:31,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:33,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:10:36,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:10:36,272 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:10:36,272 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:10:36,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:10:36,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:10:37,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:38,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:10:39,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:42,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:44,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:47,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:48,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:10:52,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:52,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:10:52,319 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:10:52,319 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:10:52,321 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:10:52,555 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:10:52,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:10:54,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:10:54,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:10:57,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:10:59,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:02,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:03,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:11:07,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:08,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:11:08,336 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:11:08,337 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:11:08,338 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:11:08,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:11:09,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:10,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:11:13,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:14,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:18,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:18,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:11:22,558 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:11:23,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:24,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:24,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:11:24,472 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:11:24,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:11:24,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:11:24,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:11:26,884 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:11:29,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:29,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:33,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:11:34,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:39,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:39,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:40,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:11:40,491 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:11:40,491 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:11:40,493 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:11:40,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:11:42,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:11:44,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:44,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:48,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:11:49,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:52,561 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:11:54,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:54,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:11:56,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:11:56,756 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:11:56,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:11:56,758 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:11:56,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:11:58,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:11:59,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:11:59,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:03,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:12:04,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:09,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:09,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:12,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:12:12,695 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:12:12,695 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:12:12,697 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:12:12,925 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:12:14,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:14,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:12:15,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:18,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:12:20,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:22,566 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:12:24,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:26,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:28,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:12:28,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:12:28,650 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:12:28,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:12:28,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:12:29,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:30,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:12:31,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:33,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:12:36,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:39,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:41,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:44,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:44,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:12:44,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:12:44,838 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:12:44,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:12:44,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:12:46,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:46,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:12:48,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:12:51,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:52,569 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:12:54,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:12:57,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:12:59,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:00,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:13:00,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:13:00,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:13:00,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:13:00,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:13:02,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:02,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:13:03,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:13:07,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:09,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:12,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:14,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:16,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:13:16,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:13:16,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:13:16,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:13:16,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:13:17,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:18,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:13:18,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:13:22,572 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:13:23,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:24,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:28,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:29,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:32,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:13:32,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:13:32,883 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:13:32,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:13:32,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:13:33,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:13:33,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:34,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:13:38,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:39,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:43,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:44,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:48,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:13:48,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:49,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:13:49,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:13:49,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:13:49,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:13:50,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:13:51,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:13:52,575 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:13:54,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:54,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:13:59,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:13:59,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:03,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:14:04,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:05,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:14:05,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:14:05,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:14:05,104 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:14:06,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:14:07,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:14:09,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:10,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:14,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:15,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:18,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:14:20,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:21,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:14:21,238 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:14:21,238 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:14:21,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:14:22,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:14:22,578 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:14:23,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:14:24,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:25,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:29,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:30,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:33,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:14:35,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:37,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:14:37,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:14:37,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:14:37,221 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:14:38,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:14:39,051 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:14:39,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:41,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:44,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:46,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:48,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:14:51,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:52,581 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:14:53,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:14:53,475 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:14:53,475 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:14:53,477 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:14:54,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:14:54,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:14:55,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:14:56,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:14:59,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:01,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:03,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:15:06,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:09,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:09,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:15:09,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:15:09,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:15:09,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:15:10,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:15:11,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:15:11,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:14,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:16,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:18,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:15:21,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:22,583 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:15:24,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:25,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:15:25,769 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:15:25,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:15:25,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:15:26,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:15:26,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:27,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:15:29,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:31,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:33,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:15:36,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:39,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:41,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:41,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:15:41,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:15:41,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:15:41,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:15:42,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:15:43,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:15:44,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:46,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:48,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:15:51,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:52,589 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:15:54,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:15:57,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:15:57,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:15:57,973 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:15:57,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:15:57,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:15:58,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:15:59,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:15:59,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:02,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:03,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:16:07,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:09,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:12,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:14,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:16:14,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:16:14,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:16:14,238 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:16:14,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:15,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:16:15,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:16:18,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:18,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:16:22,590 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:16:23,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:24,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:28,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:29,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:30,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:16:30,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:16:30,470 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:16:30,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:16:31,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:16:33,150 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:16:33,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:16:34,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:39,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:39,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:44,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:44,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:46,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:16:46,687 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:16:46,688 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:16:46,690 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:16:47,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:16:48,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:16:49,163 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:16:49,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:52,594 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:16:54,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:16:55,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:16:59,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:00,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:02,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:17:02,900 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:17:02,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:17:02,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:17:03,176 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:17:03,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:17:05,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:17:05,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:09,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:10,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:14,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:15,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:18,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:17:19,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:17:19,209 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:17:19,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:17:19,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:17:20,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:17:21,191 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:17:21,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:22,598 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:17:24,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:26,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:29,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:31,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:33,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:17:35,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:17:35,520 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:17:35,520 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:17:35,522 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:17:36,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:17:37,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:17:37,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:39,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:42,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:44,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:47,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:48,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:17:51,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:17:51,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:17:51,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:17:51,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:17:52,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:17:52,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:52,600 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:17:53,219 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:17:54,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:17:57,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:17:59,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:02,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:03,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:18:07,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:07,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:18:07,787 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:18:07,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:18:07,789 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:18:08,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:18:09,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:18:09,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:12,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:14,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:17,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:18,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:18:22,604 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:18:23,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:23,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:18:23,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:18:23,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:18:23,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:18:24,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:18:24,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:25,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:18:28,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:29,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:33,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:18:33,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:38,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:39,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:40,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:18:40,094 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:18:40,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:18:40,096 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:18:40,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:18:41,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:18:44,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:44,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:48,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:18:49,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:52,607 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:18:54,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:18:54,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:18:56,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:18:56,223 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:18:56,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:18:56,225 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:18:56,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:18:57,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:18:59,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:00,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:03,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:19:05,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:09,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:10,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:12,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:19:12,170 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:19:12,171 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:19:12,172 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:19:12,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:19:13,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:19:14,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:16,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:18,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:19:21,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:22,611 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:19:24,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:26,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:28,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:19:28,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:19:28,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:19:28,400 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:19:29,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:19:29,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:19:29,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:32,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:33,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:19:37,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:39,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:42,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:44,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:19:44,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:19:44,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:19:44,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:19:44,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:45,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:19:45,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:19:47,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:48,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:19:52,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:52,613 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:19:54,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:19:57,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:19:59,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:00,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:20:00,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:20:00,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:20:00,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:20:01,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:20:01,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:20:03,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:03,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:20:08,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:09,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:13,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:14,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:16,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:20:16,412 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:20:16,413 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:20:16,414 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:20:17,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:20:17,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:20:18,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:20:19,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:22,616 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:20:24,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:24,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:29,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:29,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:32,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:20:32,369 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:20:32,369 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:20:32,371 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:20:33,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:20:33,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:20:33,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:20:35,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:39,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:40,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:44,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:45,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:48,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:20:48,207 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:20:48,207 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:20:48,209 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:20:48,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:20:48,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:20:49,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:20:51,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:52,619 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:20:54,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:20:56,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:20:59,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:01,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:03,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:21:04,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:21:04,338 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:21:04,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:21:04,340 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:21:04,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:21:05,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:21:07,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:09,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:12,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:14,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:17,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:18,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:21:20,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:21:20,308 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:21:20,308 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:21:20,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:21:20,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:21:21,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:21:22,621 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:21:22,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:24,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:27,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:29,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:32,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:33,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:21:36,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:21:36,253 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:21:36,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:21:36,255 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:21:36,422 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:21:37,422 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:21:38,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:39,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:43,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:44,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:48,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:48,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:21:52,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:21:52,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:21:52,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:21:52,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:21:52,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:21:52,624 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:21:53,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:21:53,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:54,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:21:58,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:21:59,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:03,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:22:03,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:08,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:22:08,531 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:22:08,531 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:22:08,532 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:22:09,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:22:09,462 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:22:09,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:09,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:14,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:14,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:18,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:22:19,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:22,628 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:22:24,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:22:24,477 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:22:24,477 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:22:24,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:22:24,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:25,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:22:25,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:22:25,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:29,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:30,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:33,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:22:35,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:39,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:40,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:22:40,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:22:40,433 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:22:40,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:22:40,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:22:41,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:41,484 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:22:44,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:46,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:48,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:22:51,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:52,631 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:22:53,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:22:56,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:22:56,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:22:56,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:22:56,678 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:22:56,679 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:22:57,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:22:57,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:22:58,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:01,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:03,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:23:06,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:08,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:11,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:12,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:23:12,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:23:12,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:23:12,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:23:13,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:23:13,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:23:13,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:17,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:23:22,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:22,635 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:23:23,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:27,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:28,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:28,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:23:28,679 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:23:28,679 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:23:28,681 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:23:29,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:23:29,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:23:32,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:33,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:23:37,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:38,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:42,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:43,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:44,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:23:44,646 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:23:44,646 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:23:44,648 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:23:45,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:23:45,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:23:48,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:23:48,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:52,637 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:23:53,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:54,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:23:58,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:23:59,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:00,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:24:00,880 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:24:00,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:24:00,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:24:01,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:24:03,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:24:03,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:24:04,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:08,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:09,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:13,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:14,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:16,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:24:16,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:24:16,833 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:24:16,835 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:24:17,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:24:17,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:24:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:24:20,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:22,642 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:24:23,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:26,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:28,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:31,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:33,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:24:33,087 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:24:33,088 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:24:33,089 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:24:33,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:24:33,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:24:35,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:24:37,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:38,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:42,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:43,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:47,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:48,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:24:49,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:24:49,029 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:24:49,029 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:24:49,031 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:24:49,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:24:51,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:24:52,644 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:24:52,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:53,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:24:57,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:24:58,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:02,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:03,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:25:05,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:25:05,277 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:25:05,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:25:05,279 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:25:05,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:25:07,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:25:08,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:08,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:13,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:13,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:18,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:25:21,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:25:21,339 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:25:21,339 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:25:21,341 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:25:21,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:25:22,647 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:25:23,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:25:23,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:23,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:28,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:28,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:33,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:25:33,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:37,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:25:37,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:25:37,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:25:37,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:25:37,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:25:38,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:39,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:39,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:25:43,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:44,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:48,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:25:49,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:52,651 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:25:53,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:53,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:25:53,696 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:25:53,696 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:25:53,698 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:25:54,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:25:54,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:25:55,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:25:58,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:25:59,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:03,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:26:04,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:08,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:09,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:09,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:26:09,772 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:26:09,772 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:26:09,773 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:26:10,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:26:11,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:26:13,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:14,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:26:19,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:22,653 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:26:23,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:25,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:26,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:26:26,093 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:26:26,093 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:26:26,095 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:26:26,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:26:27,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:26:28,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:31,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:33,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:26:36,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:38,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:41,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:42,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:26:42,117 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:26:42,118 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:26:42,120 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:26:42,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:26:43,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:43,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:26:46,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:48,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:26:51,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:52,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:26:53,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:56,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:26:58,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:26:58,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:26:58,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:26:58,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:26:58,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:26:58,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:26:59,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:27:02,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:03,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:27:07,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:08,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:12,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:13,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:14,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:27:14,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:27:14,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:27:14,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:27:14,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:27:15,710 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:27:18,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:18,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:27:22,658 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:27:23,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:23,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:28,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:28,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:30,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:27:30,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:27:30,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:27:30,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:27:30,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:27:31,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:27:33,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:27:33,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:38,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:38,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:43,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:43,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:46,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:27:46,932 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:27:46,933 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:27:46,934 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:27:47,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:27:47,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:27:48,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:27:48,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:52,663 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:27:53,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:54,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:27:58,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:27:59,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:02,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:28:02,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:28:02,999 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:28:03,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:28:03,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:28:03,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:28:03,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:28:05,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:08,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:10,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:13,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:15,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:18,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:28:19,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:28:19,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:28:19,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:28:19,148 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:28:19,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:28:20,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:21,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:28:22,665 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:28:23,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:25,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:28,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:30,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:33,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:28:35,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:28:35,424 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:28:35,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:28:35,426 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:28:35,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:28:36,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:37,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:28:38,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:41,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:43,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:46,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:48,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:28:51,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:51,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:28:51,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:28:51,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:28:51,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:28:51,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:28:52,667 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:28:53,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:28:53,795 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:28:56,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:28:58,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:01,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:03,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:29:06,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:07,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:29:07,739 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:29:07,740 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:29:07,741 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:29:07,808 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:29:08,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:09,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:29:11,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:13,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:16,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:18,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:29:21,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:22,672 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:29:23,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:29:23,680 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:29:23,680 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:29:23,681 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:29:23,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:23,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:29:25,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:29:27,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:28,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:32,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:33,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:29:37,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:38,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:40,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:29:40,002 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:29:40,003 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:29:40,004 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:29:40,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:29:41,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:29:43,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:43,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:48,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:48,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:29:52,674 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:29:53,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:29:53,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:56,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:29:56,019 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:29:56,019 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:29:56,021 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:29:56,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:29:57,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:29:58,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:29:59,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:03,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:30:04,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:08,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:09,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:12,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:30:12,186 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:30:12,187 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:30:12,189 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:30:12,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:30:13,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:13,863 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:30:14,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:18,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:30:19,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:22,677 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:30:23,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:24,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:28,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:30:28,160 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:30:28,160 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:30:28,162 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:30:28,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:28,876 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:30:29,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:30:30,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:33,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:30:35,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:38,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:40,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:43,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:44,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:30:44,078 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:30:44,078 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:30:44,080 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:30:44,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:30:45,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:30:46,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:48,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:30:51,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:52,680 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:30:53,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:56,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:30:58,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:30:59,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:30:59,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:30:59,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:30:59,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:31:00,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:31:01,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:31:01,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:03,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:31:06,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:08,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:11,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:13,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:16,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:31:16,100 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:31:16,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:31:16,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:31:16,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:31:17,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:17,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:31:18,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:31:22,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:22,685 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:31:23,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:27,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:28,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:32,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:31:32,150 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:31:32,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:31:32,152 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:31:32,932 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:31:33,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:33,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:31:33,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:31:38,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:38,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:43,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:43,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:48,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:31:48,104 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:31:48,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:31:48,106 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:31:48,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:31:48,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:31:49,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:49,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:31:52,687 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:31:53,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:54,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:31:58,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:31:59,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:03,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:32:04,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:32:04,151 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:32:04,152 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:32:04,153 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:32:04,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:32:05,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:05,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:32:08,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:10,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:13,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:15,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:18,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:32:20,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:20,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:32:20,371 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:32:20,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:32:20,372 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:32:20,972 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:32:21,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:32:22,691 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:32:23,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:25,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:28,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:30,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:33,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:32:35,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:36,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:32:36,519 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:32:36,519 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:32:36,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:32:36,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:32:37,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:32:38,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:41,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:43,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:46,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:48,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:32:51,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:52,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:32:52,367 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:32:52,368 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:32:52,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:32:52,695 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:32:53,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:32:53,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:32:54,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:32:56,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:32:58,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:01,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:03,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:33:06,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:08,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:33:08,754 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:33:08,754 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:33:08,755 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:33:08,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:09,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:33:10,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:33:11,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:13,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:16,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:18,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:33:21,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:22,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:33:23,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:24,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:33:24,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:33:24,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:33:24,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:33:25,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:33:26,027 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:33:27,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:28,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:32,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:33,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:33:37,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:38,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:40,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:33:40,648 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:33:40,649 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:33:40,650 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:33:41,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:33:42,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:33:43,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:43,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:48,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:33:48,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:52,701 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:33:53,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:53,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:33:56,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:33:56,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:33:56,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:33:56,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:33:57,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:33:58,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:33:58,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:33:58,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:03,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:34:03,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:08,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:08,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:13,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:34:13,064 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:34:13,065 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:34:13,066 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:34:13,072 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:34:13,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:14,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:14,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:34:18,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:34:19,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:22,704 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:34:23,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:24,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:28,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:29,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:34:29,087 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:34:29,087 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:34:29,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:34:29,089 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:34:30,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:30,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:34:33,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:34:35,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:38,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:40,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:43,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:45,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:45,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:34:45,130 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:34:45,131 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:34:45,132 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:34:46,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:34:46,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:34:48,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:34:50,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:52,707 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:34:53,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:34:55,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:34:58,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:00,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:01,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:35:01,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:35:01,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:35:01,272 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:35:02,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:35:02,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:35:03,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:35:06,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:08,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:11,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:13,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:16,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:17,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:35:17,417 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:35:17,417 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:35:17,419 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:35:18,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:35:18,131 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:35:18,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:35:21,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:22,710 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:35:23,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:26,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:28,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:31,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:33,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:35:33,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:35:33,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:35:33,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:35:33,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:35:34,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:35:36,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:35:37,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:38,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:42,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:43,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:47,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:48,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:35:49,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:35:49,697 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:35:49,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:35:49,699 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:35:50,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:35:52,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:35:52,713 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:35:52,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:53,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:35:57,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:35:58,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:02,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:03,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:36:05,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:36:05,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:36:05,745 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:36:05,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:36:06,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:36:07,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:08,176 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:36:08,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:12,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:13,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:17,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:18,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:36:22,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:36:22,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:36:22,007 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:36:22,008 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:36:22,188 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:36:22,716 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:36:23,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:23,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:24,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:36:28,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:28,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:33,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:36:33,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:38,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:36:38,222 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:36:38,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:36:38,224 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:36:38,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:39,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:36:39,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:40,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:36:43,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:44,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:48,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:36:49,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:52,720 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:36:53,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:54,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:36:54,290 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:36:54,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:36:54,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:36:54,292 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:36:55,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:36:56,219 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:36:58,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:36:59,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:03,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:37:04,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:08,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:09,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:10,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:37:10,408 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:37:10,408 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:37:10,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:37:11,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:37:12,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:37:13,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:14,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:18,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:37:19,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:22,722 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:37:23,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:24,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:26,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:37:26,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:37:26,555 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:37:26,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:37:27,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:37:28,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:37:28,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:30,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:33,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:37:35,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:38,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:40,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:42,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:37:42,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:37:42,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:37:42,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:37:43,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:37:43,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:44,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:37:45,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:48,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:37:50,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:52,725 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:37:53,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:55,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:37:58,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:37:58,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:37:58,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:37:58,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:37:58,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:37:59,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:38:00,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:38:00,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:03,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:38:05,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:08,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:10,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:13,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:15,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:38:15,030 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:38:15,030 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:38:15,032 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:38:15,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:38:16,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:17,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:38:18,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:38:21,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:22,728 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:38:23,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:26,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:28,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:31,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:38:31,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:38:31,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:38:31,076 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:38:31,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:38:32,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:33,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:38:33,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:38:37,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:38,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:42,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:43,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:47,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:47,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:38:47,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:38:47,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:38:47,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:38:47,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:38:48,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:38:49,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:38:52,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:52,731 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:38:53,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:38:57,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:38:58,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:02,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:03,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:39:03,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:39:03,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:39:03,470 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:39:03,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:39:04,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:39:05,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:39:08,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:08,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:13,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:13,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:18,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:18,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:39:19,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:39:19,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:39:19,625 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:39:19,627 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:39:20,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:39:21,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:39:22,734 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:39:23,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:23,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:28,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:28,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:33,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:39:33,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:35,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:39:35,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:39:35,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:39:35,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:39:36,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:39:37,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:39:38,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:39,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:43,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:44,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:48,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:39:49,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:52,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:39:52,310 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:39:52,310 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:39:52,311 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:39:52,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:39:52,737 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:39:53,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:54,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:39:55,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:39:58,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:39:59,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:03,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:40:04,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:08,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:40:08,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:40:08,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:40:08,252 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:40:08,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:40:08,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:10,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:11,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:40:13,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:15,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:18,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:40:20,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:22,740 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:40:23,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:24,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:40:24,395 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:40:24,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:40:24,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:40:24,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:40:25,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:27,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:40:28,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:30,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:33,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:40:35,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:38,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:40,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:40,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:40:40,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:40:40,540 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:40:40,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:40:41,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:40:43,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:40:43,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:45,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:48,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:40:50,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:52,743 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:40:53,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:55,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:40:56,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:40:56,493 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:40:56,494 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:40:56,495 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:40:57,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:40:58,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:40:59,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:41:01,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:03,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:41:06,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:08,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:11,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:12,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:41:12,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:41:12,611 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:41:12,613 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:41:13,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:41:13,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:15,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:41:16,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:18,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:41:21,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:22,748 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:41:23,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:26,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:28,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:41:28,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:41:28,557 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:41:28,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:41:28,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:29,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:41:31,458 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:41:32,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:33,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:41:37,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:38,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:42,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:43,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:44,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:41:44,897 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:41:44,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:41:44,899 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:41:45,470 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:41:47,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:41:47,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:48,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:41:52,750 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:41:53,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:53,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:41:58,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:41:58,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:00,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:42:00,750 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:42:00,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:42:00,752 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:42:01,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:42:03,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:42:03,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:42:03,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:08,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:08,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:13,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:13,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:16,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:42:16,868 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:42:16,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:42:16,869 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:42:17,499 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:42:18,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:42:18,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:19,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:42:22,753 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:42:23,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:24,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:28,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:29,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:33,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:42:33,089 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:42:33,089 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:42:33,091 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:42:33,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:42:33,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:42:35,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:35,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:42:38,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:40,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:43,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:45,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:48,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:42:48,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:42:48,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:42:48,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:42:48,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:42:49,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:42:50,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:51,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:42:52,757 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:42:53,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:42:56,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:42:58,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:01,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:03,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:43:04,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:43:04,863 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:43:04,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:43:04,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:43:05,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:43:06,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:07,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:43:08,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:11,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:13,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:16,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:18,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:43:21,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:43:21,108 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:43:21,109 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:43:21,110 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:43:21,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:43:22,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:22,759 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:43:23,559 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:43:23,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:27,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:28,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:32,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:33,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:43:37,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:43:37,281 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:43:37,281 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:43:37,283 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:43:37,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:43:38,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:39,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:39,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:43:43,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:44,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:48,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:48,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:43:52,762 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:43:53,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:43:53,308 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:43:53,308 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:43:53,310 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:43:53,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:53,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:43:54,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:43:55,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:43:58,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:43:59,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:03,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:03,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:44:08,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:09,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:09,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:44:09,361 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:44:09,361 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:44:09,362 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:44:09,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:44:11,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:44:13,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:14,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:18,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:18,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:44:22,765 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:44:23,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:24,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:25,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:44:25,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:44:25,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:44:25,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:44:25,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:44:27,614 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:44:29,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:29,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:33,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:44:34,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:39,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:39,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:41,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:44:41,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:44:41,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:44:41,543 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:44:41,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:44:43,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:44:44,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:44,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:48,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:44:49,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:52,770 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:44:54,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:54,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:44:57,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:44:57,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:44:57,587 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:44:57,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:44:57,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:44:59,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:44:59,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:45:00,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:03,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:45:05,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:09,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:10,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:13,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:45:13,837 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:45:13,837 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:45:13,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:45:14,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:14,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:45:15,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:45:15,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:18,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:45:20,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:22,772 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:45:24,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:26,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:29,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:29,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:45:29,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:45:29,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:45:29,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:45:30,670 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:45:31,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:45:31,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:33,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:45:36,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:39,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:41,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:44,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:46,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:45:46,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:45:46,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:45:46,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:45:46,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:45:47,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:47,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:45:48,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:45:52,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:52,775 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:45:54,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:45:57,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:45:59,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:02,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:46:02,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:46:02,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:46:02,258 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:46:02,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:46:03,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:03,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:46:03,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:46:08,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:09,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:13,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:14,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:18,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:18,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:46:18,293 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:46:18,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:46:18,295 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:46:18,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:46:18,713 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:46:19,713 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:46:22,780 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:46:23,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:24,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:28,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:29,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:33,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:46:33,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:34,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:46:34,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:46:34,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:46:34,608 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:46:34,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:46:37,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:46:39,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:39,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:44,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:44,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:48,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:46:49,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:50,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:46:50,757 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:46:50,757 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:46:50,758 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:46:51,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:46:52,781 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:46:53,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:46:54,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:54,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:46:59,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:46:59,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:03,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:47:04,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:06,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:47:06,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:47:06,962 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:47:06,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:47:07,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:47:09,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:09,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:47:10,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:14,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:15,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:18,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:47:20,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:22,785 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:47:23,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:47:23,171 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:47:23,171 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:47:23,173 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:47:23,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:47:24,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:25,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:25,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:47:29,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:30,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:33,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:47:35,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:39,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:39,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:47:39,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:47:39,125 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:47:39,127 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:47:39,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:47:41,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:41,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:47:44,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:46,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:48,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:47:51,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:52,788 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:47:54,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:47:55,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:47:55,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:47:55,355 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:47:55,356 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:47:55,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:47:56,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:47:57,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:47:59,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:01,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:03,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:48:06,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:09,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:11,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:48:11,212 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:48:11,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:48:11,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:48:11,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:48:12,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:13,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:48:14,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:17,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:18,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:48:22,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:22,828 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:48:24,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:27,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:48:27,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:48:27,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:48:27,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:27,563 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:48:27,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:48:29,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:29,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:48:32,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:33,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:48:37,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:39,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:42,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:43,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:48:43,500 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:48:43,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:48:43,502 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:48:43,845 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:48:44,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:45,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:48:48,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:48,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:48:52,832 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:48:53,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:54,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:58,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:48:59,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:48:59,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:48:59,622 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:48:59,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:48:59,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:48:59,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:49:01,858 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:49:03,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:49:04,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:09,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:09,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:14,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:14,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:15,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:49:15,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:49:15,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:49:15,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:49:15,871 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:49:17,873 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:49:18,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:49:19,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:22,834 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:49:24,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:25,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:29,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:30,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:31,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:49:31,812 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:49:31,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:49:31,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:49:31,885 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:49:33,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:49:33,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:49:36,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:39,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:41,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:44,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:46,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:47,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:49:47,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:49:47,956 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:49:47,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:49:48,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:49:48,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:49:49,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:49:51,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:52,838 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:49:54,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:49:57,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:49:59,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:02,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:03,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:50:04,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:50:04,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:50:04,006 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:50:04,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:50:04,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:50:05,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:50:08,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:09,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:13,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:14,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:18,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:18,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:50:20,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:50:20,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:50:20,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:50:20,159 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:50:20,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:50:21,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:50:22,841 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:50:23,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:24,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:28,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:29,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:33,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:50:33,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:36,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:50:36,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:50:36,125 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:50:36,126 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:50:36,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:50:37,947 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:50:39,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:39,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:44,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:44,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:48,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:50:49,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:52,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:50:52,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:50:52,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:50:52,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:50:52,844 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:50:52,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:50:53,960 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:50:54,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:54,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:50:59,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:50:59,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:03,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:51:04,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:08,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:51:08,255 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:51:08,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:51:08,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:51:08,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:51:09,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:09,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:51:10,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:14,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:15,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:18,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:51:20,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:22,848 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:51:24,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:24,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:51:24,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:51:24,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:51:24,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:51:24,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:51:25,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:51:26,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:29,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:31,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:33,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:51:36,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:39,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:40,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:51:40,227 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:51:40,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:51:40,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:51:41,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:51:42,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:51:42,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:44,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:47,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:48,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:51:52,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:52,851 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:51:54,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:51:56,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:51:56,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:51:56,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:51:56,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:51:57,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:51:57,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:51:58,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:51:59,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:02,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:03,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:52:07,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:09,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:12,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:52:12,623 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:52:12,623 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:52:12,625 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:52:13,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:52:13,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:14,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:52:14,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:18,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:52:18,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:22,855 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:52:23,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:24,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:28,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:52:28,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:52:28,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:52:28,765 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:52:29,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:52:29,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:29,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:30,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:52:33,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:52:34,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:39,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:39,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:44,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:44,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:44,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:52:44,981 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:52:44,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:52:44,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:52:45,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:52:46,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:52:48,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:52:49,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:52,857 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:52:54,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:52:55,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:52:59,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:00,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:00,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:53:00,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:53:00,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:53:00,929 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:53:01,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:53:02,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:53:03,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:53:05,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:09,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:10,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:14,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:15,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:16,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:53:16,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:53:16,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:53:16,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:53:17,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:53:18,096 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:53:18,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:53:20,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:22,862 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:53:24,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:26,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:29,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:31,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:33,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:53:33,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:53:33,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:53:33,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:53:33,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:53:34,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:53:34,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:53:37,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:39,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:42,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:44,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:47,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:48,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:53:49,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:53:49,372 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:53:49,372 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:53:49,374 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:53:50,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:53:52,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:53:52,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:52,864 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:53:54,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:53:57,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:53:59,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:02,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:03,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:54:05,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:54:05,241 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:54:05,241 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:54:05,243 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:54:06,137 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:54:06,137 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:54:08,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:09,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:13,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:14,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:18,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:18,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:54:21,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:54:21,376 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:54:21,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:54:21,378 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:54:22,151 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:54:22,867 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:54:23,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:24,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:24,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:54:28,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:29,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:33,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:54:33,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:37,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:54:37,614 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:54:37,615 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:54:37,616 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:54:38,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:54:39,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:39,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:40,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:54:44,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:44,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:48,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:54:49,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:52,869 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:54:53,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:54:53,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:54:53,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:54:53,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:54:54,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:54,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:54:54,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:54:56,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:54:59,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:54:59,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:03,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:55:04,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:09,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:09,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:55:09,831 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:55:09,831 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:55:09,833 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:55:10,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:55:10,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:12,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:55:14,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:15,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:18,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:55:20,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:22,873 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:55:24,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:25,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:26,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:55:26,056 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:55:26,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:55:26,058 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:55:26,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:55:28,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:55:29,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:31,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:33,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:55:36,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:39,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:41,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:42,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:55:42,098 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:55:42,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:55:42,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:55:42,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:55:44,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:44,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:55:46,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:48,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:55:51,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:52,876 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:55:54,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:56,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:55:58,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:55:58,349 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:55:58,349 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:55:58,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:55:59,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:55:59,240 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:56:00,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:56:02,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:03,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:56:07,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:09,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:12,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:14,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:14,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:56:14,406 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:56:14,406 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:56:14,408 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:56:15,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:56:16,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:56:17,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:18,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:56:22,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:22,879 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:56:24,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:27,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:29,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:30,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:56:30,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:56:30,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:56:30,668 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:56:31,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:56:32,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:56:33,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:56:33,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:38,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:39,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:43,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:44,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:46,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:56:46,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:56:46,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:56:46,803 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:56:47,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:56:48,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:56:48,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:56:48,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:52,884 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:56:53,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:54,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:56:58,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:56:59,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:03,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:57:03,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:57:03,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:57:03,042 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:57:03,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:57:03,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:57:04,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:04,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:57:09,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:09,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:14,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:14,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:18,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:57:19,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:19,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:57:19,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:57:19,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:57:19,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:57:19,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:57:20,312 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:57:22,886 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:57:24,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:24,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:29,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:29,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:33,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:57:34,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:35,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:57:35,275 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:57:35,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:57:35,277 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:57:35,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:57:36,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:57:39,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:40,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:44,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:45,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:48,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:57:50,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:51,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:57:51,315 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:57:51,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:57:51,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:57:51,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:57:52,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:57:52,889 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:57:54,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:57:55,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:57:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:00,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:03,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:58:05,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:07,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:58:07,479 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:58:07,479 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:58:07,480 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:58:08,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:58:08,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:58:09,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:11,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:14,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:16,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:18,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:58:21,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:22,893 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:58:23,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:58:23,429 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:58:23,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:58:23,431 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:58:24,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:24,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:58:24,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:58:27,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:29,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:32,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:33,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:58:37,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:39,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:39,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:58:39,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:58:39,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:58:39,608 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:58:40,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:58:40,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:58:42,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:44,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:47,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:48,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:58:52,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:52,895 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:58:54,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:58:55,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:58:55,656 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:58:55,656 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:58:55,658 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:58:56,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:58:56,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:58:57,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:58:59,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:02,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:03,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:59:07,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:09,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:11,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:59:11,609 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:59:11,610 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:59:11,611 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:59:12,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:59:12,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:59:13,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:14,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:18,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:59:18,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:22,899 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:59:23,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:24,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:27,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:59:27,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:59:27,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:59:27,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:59:28,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:59:29,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:29,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:30,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:59:33,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:59:34,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:39,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:39,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:43,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:59:43,933 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:59:43,934 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:59:43,935 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 10:59:44,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:44,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 10:59:44,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:46,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 10:59:48,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 10:59:49,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:52,903 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 10:59:54,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:55,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 10:59:59,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 10:59:59,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 10:59:59,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 10:59:59,971 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 10:59:59,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:00:00,452 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:00:00,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:02,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:00:03,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:00:05,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:09,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:10,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:14,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:15,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:16,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:00:16,057 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:00:16,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:00:16,059 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:00:16,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:00:18,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:00:18,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:00:21,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:22,906 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:00:24,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:26,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:29,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:31,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:32,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:00:32,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:00:32,098 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:00:32,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:00:32,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:00:33,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:00:34,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:00:37,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:39,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:42,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:44,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:47,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:48,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:00:48,064 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:00:48,064 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:00:48,065 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:00:48,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:00:48,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:00:50,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:00:52,908 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:00:52,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:54,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:00:57,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:00:59,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:02,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:03,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:01:04,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:01:04,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:01:04,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:01:04,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:01:04,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:01:06,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:08,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:08,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:09,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:12,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:14,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:14,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:15,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:16,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:18,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:01:19,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:20,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:22,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:22,912 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:01:24,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:01:24,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:25,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:26,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:29,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:30,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:33,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:01:35,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:39,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:40,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:40,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:01:40,438 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:01:40,438 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:01:40,440 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:01:40,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:01:42,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:44,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:45,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:48,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:01:50,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:52,917 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:01:54,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:01:55,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:01:56,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:01:56,653 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:01:56,653 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:01:56,654 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:01:57,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:01:58,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:01:59,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:01,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:03,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:02:06,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:09,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:11,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:12,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:02:12,945 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:02:12,945 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:02:12,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:02:13,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:02:14,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:14,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:02:16,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:18,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:02:21,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:22,918 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:02:24,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:27,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:28,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:02:28,874 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:02:28,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:02:28,876 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:02:29,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:29,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:02:30,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:02:33,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:02:33,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:38,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:39,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:43,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:44,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:45,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:02:45,188 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:02:45,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:02:45,190 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:02:45,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:02:46,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:02:48,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:02:49,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:52,924 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:02:54,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:54,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:02:59,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:02:59,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:01,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:03:01,494 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:03:01,494 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:03:01,495 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:03:01,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:03:02,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:03:03,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:03:05,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:09,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:10,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:14,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:15,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:17,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:03:17,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:03:17,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:03:17,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:03:17,630 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:03:18,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:03:18,631 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:03:21,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:22,929 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:03:24,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:26,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:29,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:31,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:33,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:03:33,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:03:33,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:03:33,402 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:03:33,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:03:33,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:03:34,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:03:37,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:39,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:42,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:44,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:47,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:48,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:03:49,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:03:49,378 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:03:49,378 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:03:49,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:03:49,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:03:50,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:03:52,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:52,931 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:03:54,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:03:57,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:03:59,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:02,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:03,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:04:05,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:04:05,230 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:04:05,231 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:04:05,232 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:04:05,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:04:06,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:04:08,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:09,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:13,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:14,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:18,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:18,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:04:21,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:04:21,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:04:21,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:04:21,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:04:21,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:04:22,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:04:22,934 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:04:23,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:24,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:28,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:29,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:33,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:04:33,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:37,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:04:37,219 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:04:37,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:04:37,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:04:37,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:04:38,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:04:39,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:39,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:44,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:44,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:48,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:04:49,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:52,967 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:04:53,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:04:53,495 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:04:53,496 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:04:53,497 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:04:53,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:04:54,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:54,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:04:54,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:04:59,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:04:59,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:03,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:05:04,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:09,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:09,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:09,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:05:09,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:05:09,677 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:05:09,678 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:05:09,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:05:10,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:05:14,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:14,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:18,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:05:19,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:22,971 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:05:24,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:24,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:25,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:05:25,646 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:05:25,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:05:25,648 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:05:25,732 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:05:26,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:05:29,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:30,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:33,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:05:35,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:39,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:40,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:41,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:05:41,645 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:05:41,645 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:05:41,646 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:05:41,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:05:42,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:05:44,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:46,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:48,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:05:51,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:52,975 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:05:54,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:05:56,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:05:57,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:05:57,817 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:05:57,818 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:05:57,819 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:05:58,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:05:58,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:05:59,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:02,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:03,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:06:07,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:09,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:12,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:13,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:06:13,680 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:06:13,680 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:06:13,682 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:06:13,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:06:14,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:14,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:06:18,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:06:18,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:22,983 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:06:23,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:24,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:28,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:29,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:29,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:06:29,751 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:06:29,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:06:29,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:06:29,787 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:06:30,788 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:06:33,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:06:34,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:39,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:39,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:44,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:44,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:45,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:06:45,898 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:06:45,899 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:06:45,900 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:06:46,801 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:06:46,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:06:48,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:06:49,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:52,988 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:06:54,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:54,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:06:59,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:06:59,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:01,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:07:01,854 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:07:01,854 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:07:01,855 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:07:02,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:07:02,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:07:03,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:07:05,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:09,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:10,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:14,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:15,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:17,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:07:17,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:07:17,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:07:17,915 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:07:18,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:07:18,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:07:18,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:07:20,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:22,989 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:07:24,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:25,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:29,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:30,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:33,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:07:34,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:07:34,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:07:34,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:07:34,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:07:34,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:07:34,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:07:36,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:39,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:41,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:44,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:46,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:48,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:07:50,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:07:50,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:07:50,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:07:50,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:07:50,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:07:51,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:52,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:07:52,993 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:07:54,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:07:56,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:07:59,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:01,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:03,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:08:06,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:08:06,145 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:08:06,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:08:06,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:08:06,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:08:06,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:08:07,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:09,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:12,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:14,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:17,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:18,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:08:22,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:22,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:08:22,302 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:08:22,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:08:22,304 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:08:22,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:08:22,997 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:08:24,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:24,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:08:27,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:29,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:32,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:33,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:08:38,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:38,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:08:38,521 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:08:38,521 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:08:38,523 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:08:38,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:08:39,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:40,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:08:43,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:44,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:48,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:08:48,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:52,999 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:08:54,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:08:54,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:54,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:08:54,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:08:54,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:08:54,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:08:54,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:08:56,910 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:08:59,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:08:59,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:03,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:09:04,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:09,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:09,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:10,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:09:10,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:09:10,600 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:09:10,601 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:09:10,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:09:12,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:09:14,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:14,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:18,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:09:19,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:23,003 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:09:24,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:25,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:27,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:09:27,030 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:09:27,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:09:27,033 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:09:27,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:09:28,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:09:29,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:30,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:33,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:09:35,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:39,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:40,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:42,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:09:42,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:09:42,989 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:09:42,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:09:43,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:09:44,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:44,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:09:45,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:48,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:09:50,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:53,006 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:09:54,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:56,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:09:59,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:09:59,241 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:09:59,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:09:59,243 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:09:59,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:09:59,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:10:00,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:10:01,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:03,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:10:06,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:09,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:11,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:14,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:15,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:10:15,441 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:10:15,441 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:10:15,443 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:10:15,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:10:16,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:16,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:10:18,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:10:21,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:23,009 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:10:24,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:27,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:29,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:31,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:10:31,658 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:10:31,658 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:10:31,660 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:10:31,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:10:32,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:32,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:10:33,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:10:37,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:39,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:42,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:44,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:47,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:47,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:10:47,773 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:10:47,774 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:10:47,775 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:10:48,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:10:48,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:10:49,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:10:52,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:53,013 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:10:54,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:10:58,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:10:59,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:03,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:03,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:11:04,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:11:04,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:11:04,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:11:04,006 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:11:04,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:11:05,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:11:08,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:09,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:13,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:14,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:18,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:18,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:11:20,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:11:20,222 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:11:20,222 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:11:20,224 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:11:21,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:11:21,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:11:23,016 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:11:24,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:24,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:29,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:29,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:33,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:11:34,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:36,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:11:36,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:11:36,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:11:36,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:11:37,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:11:37,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:11:39,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:39,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:44,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:44,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:48,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:11:49,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:52,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:11:52,541 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:11:52,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:11:52,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:11:53,019 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:11:53,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:11:54,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:11:55,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:11:55,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:11:59,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:00,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:03,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:12:05,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:08,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:12:08,653 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:12:08,654 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:12:08,655 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:12:09,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:12:09,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:10,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:11,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:12:14,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:15,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:18,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:12:20,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:23,022 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:12:24,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:24,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:12:24,897 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:12:24,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:12:24,900 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:12:25,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:12:25,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:27,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:12:29,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:30,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:33,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:12:35,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:39,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:40,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:40,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:12:40,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:12:40,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:12:40,925 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:12:41,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:12:43,101 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:12:44,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:45,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:48,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:12:50,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:53,026 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:12:54,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:12:56,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:12:57,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:12:57,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:12:57,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:12:57,134 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:12:58,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:12:59,114 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:12:59,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:01,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:03,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:13:06,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:09,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:11,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:13,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:13:13,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:13:13,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:13:13,159 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:13:14,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:13:14,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:15,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:13:16,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:18,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:13:21,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:23,028 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:13:24,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:27,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:29,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:13:29,183 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:13:29,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:13:29,185 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:13:29,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:30,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:13:31,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:13:32,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:33,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:13:37,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:39,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:42,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:44,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:45,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:13:45,221 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:13:45,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:13:45,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:13:46,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:13:47,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:13:47,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:48,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:13:52,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:53,031 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:13:54,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:13:58,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:13:59,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:01,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:14:01,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:14:01,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:14:01,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:14:01,168 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:14:03,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:03,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:14:03,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:14:08,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:09,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:13,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:14,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:17,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:14:17,137 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:14:17,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:14:17,139 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:14:17,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:14:18,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:14:19,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:19,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:14:23,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:14:24,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:25,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:29,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:30,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:33,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:14:33,183 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:14:33,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:14:33,185 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:14:33,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:14:33,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:14:35,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:35,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:14:39,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:40,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:44,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:45,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:48,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:14:49,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:14:49,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:14:49,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:14:49,124 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:14:49,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:14:51,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:51,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:14:53,037 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:14:54,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:14:57,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:14:59,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:02,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:03,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:15:05,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:15:05,374 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:15:05,375 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:15:05,376 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:15:06,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:15:07,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:15:07,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:09,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:12,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:14,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:17,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:18,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:15:21,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:15:21,415 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:15:21,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:15:21,418 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:15:22,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:15:22,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:23,042 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:15:23,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:15:24,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:28,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:29,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:33,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:33,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:15:37,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:15:37,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:15:37,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:15:37,381 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:15:38,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:15:38,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:39,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:15:39,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:43,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:44,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:48,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:48,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:15:53,043 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:15:53,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:15:53,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:15:53,536 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:15:53,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:53,538 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:15:54,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:15:54,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:15:55,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:15:58,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:15:59,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:03,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:16:03,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:08,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:09,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:09,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:16:09,575 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:16:09,576 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:16:09,577 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:16:10,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:16:11,281 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:16:14,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:14,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:18,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:16:19,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:23,047 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:16:24,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:25,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:25,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:16:25,522 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:16:25,523 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:16:25,524 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:16:26,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:16:27,294 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:16:29,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:30,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:33,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:16:35,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:39,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:40,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:41,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:16:41,569 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:16:41,570 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:16:41,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:16:42,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:16:43,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:16:44,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:45,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:48,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:16:50,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:53,051 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:16:54,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:16:56,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:16:57,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:16:57,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:16:57,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:16:57,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:16:58,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:16:59,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:16:59,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:01,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:03,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:17:06,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:09,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:11,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:13,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:17:13,739 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:17:13,740 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:17:13,741 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:17:14,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:17:14,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:15,335 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:17:16,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:18,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:17:21,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:23,053 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:17:24,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:27,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:29,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:29,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:17:29,702 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:17:29,702 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:17:29,704 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:17:30,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:17:31,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:17:32,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:33,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:17:37,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:39,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:42,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:44,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:45,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:17:45,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:17:45,927 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:17:45,929 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:17:46,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:17:47,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:17:47,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:48,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:17:52,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:53,055 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:17:54,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:17:58,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:17:59,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:02,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:18:02,060 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:18:02,061 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:18:02,062 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:18:02,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:18:03,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:03,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:18:03,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:18:08,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:09,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:13,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:14,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:18,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:18:18,028 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:18:18,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:18:18,030 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:18:18,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:18:18,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:18:19,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:19,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:18:23,059 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:18:24,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:24,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:29,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:29,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:33,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:18:34,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:34,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:18:34,181 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:18:34,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:18:34,182 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:18:34,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:18:35,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:18:39,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:39,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:44,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:44,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:48,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:18:49,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:50,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:18:50,130 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:18:50,130 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:18:50,131 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:18:50,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:18:51,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:18:53,063 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:18:54,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:18:55,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:18:59,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:00,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:03,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:19:05,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:06,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:19:06,367 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:19:06,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:19:06,369 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:19:06,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:19:07,432 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:19:09,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:10,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:14,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:15,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:18,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:19:20,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:22,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:19:22,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:19:22,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:19:22,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:19:22,449 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:19:23,066 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:19:23,449 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:19:24,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:26,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:29,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:31,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:33,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:19:36,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:38,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:19:38,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:19:38,556 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:19:38,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:19:39,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:19:39,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:19:39,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:41,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:44,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:46,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:48,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:19:51,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:53,068 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:19:54,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:19:54,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:19:54,645 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:19:54,645 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:19:54,647 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:19:55,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:19:55,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:19:56,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:19:59,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:01,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:03,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:20:06,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:09,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:10,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:20:10,912 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:20:10,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:20:10,914 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:20:11,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:20:11,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:20:11,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:14,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:16,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:18,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:20:21,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:23,073 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:20:24,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:27,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:27,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:20:27,168 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:20:27,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:20:27,170 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:20:27,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:20:29,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:20:29,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:32,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:33,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:20:37,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:39,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:42,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:43,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:20:43,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:20:43,277 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:20:43,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:20:43,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:20:44,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:45,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:20:47,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:48,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:20:52,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:53,074 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:20:54,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:58,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:20:59,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:20:59,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:20:59,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:20:59,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:20:59,629 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:21:00,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:21:01,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:21:03,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:21:03,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:08,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:09,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:13,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:14,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:15,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:21:15,754 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:21:15,754 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:21:15,756 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:21:16,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:21:17,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:21:18,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:21:18,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:23,077 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:21:24,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:24,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:29,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:29,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:32,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:21:32,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:21:32,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:21:32,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:21:32,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:21:33,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:21:33,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:21:35,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:39,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:40,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:44,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:45,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:48,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:21:48,036 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:21:48,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:21:48,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:21:48,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:21:48,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:21:49,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:21:51,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:53,080 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:21:54,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:21:56,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:21:59,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:01,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:03,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:22:04,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:22:04,208 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:22:04,208 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:22:04,209 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:22:04,591 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:22:05,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:22:06,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:09,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:11,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:14,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:16,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:18,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:22:20,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:22:20,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:22:20,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:22:20,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:22:20,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:22:21,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:21,605 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:22:23,083 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:22:24,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:27,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:29,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:32,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:33,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:22:36,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:22:36,622 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:22:36,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:22:36,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:22:37,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:22:37,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:22:37,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:39,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:42,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:44,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:47,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:48,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:22:52,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:22:52,659 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:22:52,659 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:22:52,661 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:22:52,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:53,113 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:22:53,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:22:53,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:22:54,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:22:58,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:22:59,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:03,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:03,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:23:08,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:08,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:23:08,937 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:23:08,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:23:08,939 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:23:09,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:09,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:23:09,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:23:13,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:14,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:18,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:23:18,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:23,114 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:23:24,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:24,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:24,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:23:24,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:23:24,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:23:24,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:23:25,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:23:25,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:23:29,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:29,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:33,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:23:34,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:39,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:39,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:41,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:23:41,228 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:23:41,229 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:23:41,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:23:41,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:23:41,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:23:44,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:45,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:48,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:23:50,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:53,120 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:23:54,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:23:56,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:23:57,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:23:57,168 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:23:57,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:23:57,170 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:23:57,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:23:57,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:23:59,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:01,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:03,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:24:06,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:09,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:11,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:13,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:24:13,207 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:24:13,207 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:24:13,209 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:24:13,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:24:13,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:24:14,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:16,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:18,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:24:21,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:23,121 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:24:24,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:27,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:29,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:24:29,053 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:24:29,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:24:29,055 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:24:29,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:29,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:24:29,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:24:33,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:33,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:24:38,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:39,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:43,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:44,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:45,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:24:45,109 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:24:45,109 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:24:45,111 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:24:45,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:24:45,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:24:48,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:48,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:24:53,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:53,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:24:54,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:24:58,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:24:59,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:01,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:25:01,036 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:25:01,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:25:01,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:25:01,745 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:25:01,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:25:03,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:25:04,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:09,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:09,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:14,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:14,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:17,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:25:17,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:25:17,402 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:25:17,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:25:17,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:25:17,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:25:18,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:25:19,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:23,128 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:25:24,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:25,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:29,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:30,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:33,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:25:33,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:25:33,461 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:25:33,462 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:25:33,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:25:33,776 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:25:35,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:35,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:25:39,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:40,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:44,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:45,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:48,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:25:49,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:25:49,464 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:25:49,464 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:25:49,466 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:25:49,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:25:50,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:51,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:25:53,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:25:54,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:25:56,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:25:59,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:01,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:03,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:26:05,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:26:05,509 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:26:05,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:26:05,511 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:26:05,804 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:26:06,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:07,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:26:09,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:11,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:14,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:16,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:18,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:26:21,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:26:21,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:26:21,536 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:26:21,537 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:26:21,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:21,817 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:26:23,134 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:26:23,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:26:24,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:27,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:29,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:32,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:33,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:26:37,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:37,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:26:37,570 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:26:37,571 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:26:37,572 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:26:37,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:26:39,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:39,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:26:42,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:44,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:47,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:48,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:26:52,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:53,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:26:53,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:26:53,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:26:53,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:26:53,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:26:53,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:26:53,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:26:55,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:26:58,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:26:58,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:03,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:03,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:27:08,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:08,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:09,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:27:09,548 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:27:09,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:27:09,550 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:27:09,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:27:11,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:27:13,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:14,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:18,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:27:19,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:23,141 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:27:23,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:25,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:25,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:27:25,501 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:27:25,502 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:27:25,503 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:27:25,876 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:27:25,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:27:28,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:30,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:33,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:27:35,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:38,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:40,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:41,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:27:41,574 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:27:41,575 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:27:41,577 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:27:41,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:27:43,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:43,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:27:45,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:48,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:27:50,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:53,143 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:27:53,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:56,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:27:57,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:27:57,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:27:57,727 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:27:57,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:27:57,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:27:58,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:27:59,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:28:01,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:03,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:28:06,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:08,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:11,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:13,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:13,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:28:13,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:28:13,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:28:13,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:28:13,920 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:28:15,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:28:16,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:18,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:28:21,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:23,147 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:28:23,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:27,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:28,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:29,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:28:29,772 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:28:29,772 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:28:29,774 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:28:29,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:28:31,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:28:32,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:33,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:28:37,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:38,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:42,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:43,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:46,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:28:46,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:28:46,007 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:28:46,008 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:28:46,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:28:47,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:28:48,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:48,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:28:53,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:53,150 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:28:53,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:28:58,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:28:58,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:01,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:29:01,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:29:01,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:29:01,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:29:01,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:29:03,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:29:03,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:03,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:29:08,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:08,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:13,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:13,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:18,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:29:18,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:29:18,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:29:18,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:29:18,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:29:18,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:29:19,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:19,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:29:23,152 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:29:23,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:25,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:28,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:30,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:33,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:29:34,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:29:34,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:29:34,132 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:29:34,133 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:29:34,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:29:35,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:35,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:29:38,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:40,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:43,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:45,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:48,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:29:50,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:50,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:29:50,494 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:29:50,495 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:29:50,496 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:29:51,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:29:52,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:29:53,155 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:29:53,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:29:56,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:29:58,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:01,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:03,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:30:06,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:06,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:30:06,481 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:30:06,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:30:06,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:30:07,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:30:08,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:30:08,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:11,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:13,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:16,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:18,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:30:21,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:22,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:30:22,904 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:30:22,905 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:30:22,906 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:30:23,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:30:23,158 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:30:23,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:24,039 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:30:27,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:28,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:32,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:33,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:30:37,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:38,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:39,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:30:39,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:30:39,049 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:30:39,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:30:39,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:30:40,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:30:43,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:43,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:48,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:48,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:30:53,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:53,162 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:30:53,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:30:55,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:30:55,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:30:55,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:30:55,294 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:30:56,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:30:56,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:30:58,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:30:58,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:03,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:03,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:31:08,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:08,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:11,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:31:11,515 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:31:11,515 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:31:11,516 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:31:12,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:31:12,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:31:13,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:13,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:18,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:18,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:31:23,164 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:31:23,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:24,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:27,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:31:27,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:31:27,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:31:27,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:31:28,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:31:28,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:31:28,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:29,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:33,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:31:34,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:38,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:39,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:43,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:43,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:31:43,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:31:43,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:31:43,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:31:44,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:31:44,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:46,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:31:48,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:31:49,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:53,167 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:31:53,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:31:55,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:31:58,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:00,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:32:00,014 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:32:00,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:32:00,016 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:32:00,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:32:01,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:02,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:32:03,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:32:06,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:08,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:11,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:13,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:16,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:16,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:32:16,153 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:32:16,153 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:32:16,155 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:32:17,141 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:32:18,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:32:18,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:32:21,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:23,171 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:32:23,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:26,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:28,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:31,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:32,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:32:32,268 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:32:32,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:32:32,270 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:32:33,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:32:33,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:32:34,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:32:36,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:38,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:41,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:43,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:46,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:48,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:32:48,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:32:48,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:32:48,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:32:48,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:32:49,164 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:32:50,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:32:51,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:53,174 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:32:53,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:32:57,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:32:58,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:02,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:03,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:33:04,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:33:04,344 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:33:04,344 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:33:04,345 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:33:05,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:33:06,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:33:07,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:08,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:12,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:13,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:17,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:18,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:33:20,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:33:20,574 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:33:20,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:33:20,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:33:21,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:33:22,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:33:22,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:23,178 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:33:23,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:28,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:28,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:33,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:33,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:33:36,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:33:36,588 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:33:36,588 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:33:36,589 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:33:37,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:33:38,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:33:38,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:38,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:43,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:43,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:48,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:33:48,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:52,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:33:52,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:33:52,820 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:33:52,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:33:53,182 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:33:53,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:33:53,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:54,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:33:54,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:33:58,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:33:59,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:03,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:34:04,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:08,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:08,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:34:08,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:34:08,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:34:08,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:34:09,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:34:09,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:10,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:34:13,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:14,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:18,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:34:19,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:23,183 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:34:23,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:24,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:34:24,836 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:34:24,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:34:24,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:24,838 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:34:25,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:34:26,251 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:34:28,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:29,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:33,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:34:34,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:38,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:39,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:40,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:34:40,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:34:40,818 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:34:40,819 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:34:41,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:34:42,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:34:43,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:45,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:48,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:34:50,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:53,187 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:34:53,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:34:56,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:34:56,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:34:56,868 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:34:56,868 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:34:56,870 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:34:57,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:34:58,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:34:58,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:01,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:03,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:35:06,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:08,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:11,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:12,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:35:12,822 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:35:12,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:35:12,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:35:13,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:35:13,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:14,293 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:35:17,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:18,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:35:22,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:23,190 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:35:23,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:28,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:28,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:29,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:35:29,144 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:35:29,144 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:35:29,146 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:35:29,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:35:30,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:35:33,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:35:34,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:38,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:39,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:43,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:44,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:45,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:35:45,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:35:45,167 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:35:45,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:35:45,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:35:46,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:35:48,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:35:49,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:53,195 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:35:53,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:54,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:35:58,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:35:59,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:01,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:36:01,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:36:01,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:36:01,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:36:01,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:36:02,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:36:03,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:36:04,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:08,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:09,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:13,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:14,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:17,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:36:17,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:36:17,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:36:17,244 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:36:17,352 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:36:18,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:36:18,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:36:19,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:23,200 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:36:23,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:25,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:28,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:30,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:33,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:36:33,479 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:36:33,480 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:36:33,482 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:36:33,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:36:34,370 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:36:34,370 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:36:35,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:38,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:40,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:43,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:45,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:48,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:36:49,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:36:49,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:36:49,419 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:36:49,420 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:36:50,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:36:50,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:36:51,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:53,200 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:36:53,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:36:57,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:36:58,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:02,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:03,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:37:05,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:37:05,269 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:37:05,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:37:05,272 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:37:05,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:37:06,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:37:07,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:08,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:12,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:13,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:17,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:18,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:37:21,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:37:21,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:37:21,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:37:21,321 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:37:21,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:37:22,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:22,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:37:23,204 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:37:23,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:28,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:28,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:33,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:33,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:37:37,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:37:37,462 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:37:37,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:37:37,464 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:37:38,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:37:38,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:37:38,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:38,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:43,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:43,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:48,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:48,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:37:53,206 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:37:53,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:37:53,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:37:53,420 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:37:53,421 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:37:53,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:37:53,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:54,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:37:54,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:37:58,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:37:59,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:03,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:38:04,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:08,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:09,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:09,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:38:09,468 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:38:09,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:38:09,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:38:10,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:38:10,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:38:13,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:14,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:18,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:38:19,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:23,209 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:38:23,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:25,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:25,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:38:25,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:38:25,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:38:25,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:38:26,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:38:26,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:38:28,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:30,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:33,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:38:35,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:38,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:40,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:41,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:38:41,592 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:38:41,592 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:38:41,595 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:38:42,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:38:42,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:38:43,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:46,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:48,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:38:51,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:53,212 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:38:53,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:38:57,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:38:57,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:38:57,971 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:38:57,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:38:57,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:38:58,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:38:58,498 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:38:58,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:02,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:03,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:39:07,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:08,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:12,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:13,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:39:13,843 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:39:13,844 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:39:13,846 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:39:13,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:14,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:39:14,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:39:18,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:39:18,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:23,215 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:39:23,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:24,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:28,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:29,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:30,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:39:30,190 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:39:30,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:39:30,192 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:39:30,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:39:30,526 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:39:33,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:39:35,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:38,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:40,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:43,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:45,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:46,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:39:46,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:39:46,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:39:46,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:39:46,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:39:46,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:39:48,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:39:51,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:53,218 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:39:53,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:39:56,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:39:58,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:01,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:02,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:40:02,482 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:40:02,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:40:02,484 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:40:02,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:40:03,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:40:04,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:40:06,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:08,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:11,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:13,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:16,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:18,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:40:18,555 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:40:18,555 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:40:18,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:40:18,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:40:18,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:40:20,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:40:21,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:23,220 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:40:23,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:27,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:28,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:32,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:33,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:40:34,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:40:34,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:40:34,691 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:40:34,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:40:35,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:40:36,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:40:37,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:38,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:42,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:43,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:47,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:48,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:40:50,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:40:50,949 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:40:50,949 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:40:50,951 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:40:51,600 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:40:52,601 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:40:52,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:53,225 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:40:53,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:40:58,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:40:58,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:03,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:03,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:41:07,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:41:07,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:41:07,169 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:41:07,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:41:07,615 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:41:08,616 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:41:08,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:09,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:13,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:14,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:18,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:41:19,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:23,257 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:41:23,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:41:23,491 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:41:23,491 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:41:23,492 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:41:23,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:41:23,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:24,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:24,629 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:41:28,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:29,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:33,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:41:34,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:38,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:39,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:39,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:41:39,561 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:41:39,562 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:41:39,563 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:41:39,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:41:40,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:41:43,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:44,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:48,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:41:49,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:53,260 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:41:53,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:41:55,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:41:56,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:41:56,013 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:41:56,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:41:56,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:41:56,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:41:56,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:41:58,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:01,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:03,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:42:06,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:08,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:11,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:11,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:42:11,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:42:11,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:42:11,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:42:12,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:42:12,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:42:13,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:16,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:42:21,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:23,264 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:42:23,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:27,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:28,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:42:28,273 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:42:28,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:42:28,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:42:28,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:42:28,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:42:28,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:32,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:33,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:42:37,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:38,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:42,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:43,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:44,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:42:44,243 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:42:44,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:42:44,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:42:44,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:42:44,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:42:48,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:48,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:42:53,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:53,267 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:42:53,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:42:58,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:42:58,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:00,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:43:00,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:43:00,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:43:00,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:43:00,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:43:00,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:43:03,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:03,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:43:08,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:08,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:13,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:13,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:16,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:43:16,435 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:43:16,435 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:43:16,437 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:43:16,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:43:16,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:43:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:43:19,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:23,270 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:43:23,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:25,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:28,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:30,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:32,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:43:32,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:43:32,692 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:43:32,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:43:32,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:43:33,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:43:34,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:43:35,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:38,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:40,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:43,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:45,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:48,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:43:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:43:48,715 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:43:48,716 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:43:48,717 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:43:48,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:43:50,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:43:50,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:53,273 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:43:53,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:43:56,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:43:58,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:01,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:03,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:44:05,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:44:05,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:44:05,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:44:05,063 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:44:05,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:44:06,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:44:07,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:08,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:12,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:13,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:17,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:44:21,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:44:21,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:44:21,004 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:44:21,006 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:44:21,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:44:22,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:44:23,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:23,277 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:44:23,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:28,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:28,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:33,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:33,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:44:37,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:44:37,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:44:37,041 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:44:37,042 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:44:37,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:44:38,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:44:38,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:39,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:43,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:44,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:48,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:44:49,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:52,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:44:52,984 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:44:52,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:44:52,986 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:44:53,279 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:44:53,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:44:53,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:54,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:44:54,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:44:58,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:44:59,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:03,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:45:04,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:08,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:09,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:45:09,035 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:45:09,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:45:09,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:45:09,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:45:10,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:10,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:45:13,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:15,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:18,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:45:20,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:23,282 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:45:23,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:25,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:45:25,067 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:45:25,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:45:25,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:25,069 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:45:25,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:45:26,841 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:45:28,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:30,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:33,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:45:35,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:38,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:40,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:41,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:45:41,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:45:41,135 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:45:41,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:45:41,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:45:43,855 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:45:43,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:45,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:48,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:45:50,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:53,284 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:45:53,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:55,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:45:57,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:45:57,717 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:45:57,718 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:45:57,719 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:45:57,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:45:58,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:45:59,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:46:00,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:03,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:46:05,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:08,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:10,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:13,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:46:13,761 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:46:13,761 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:46:13,763 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:46:13,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:46:13,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:15,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:15,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:46:18,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:46:20,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:23,287 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:46:23,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:26,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:28,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:29,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:46:29,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:46:29,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:46:29,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:46:29,896 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:46:31,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:31,898 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:46:33,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:46:36,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:38,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:41,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:43,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:45,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:46:45,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:46:45,769 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:46:45,770 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:46:45,910 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:46:46,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:47,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:46:48,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:46:51,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:53,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:46:53,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:46:57,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:46:58,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:01,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:47:01,722 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:47:01,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:47:01,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:47:01,923 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:47:02,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:03,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:47:03,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:47:07,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:08,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:12,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:13,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:17,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:47:17,568 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:47:17,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:47:17,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:47:17,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:47:18,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:18,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:47:19,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:47:23,294 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:47:23,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:24,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:28,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:29,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:33,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:47:33,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:47:33,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:47:33,851 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:47:33,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:47:33,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:47:34,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:35,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:47:38,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:39,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:43,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:44,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:48,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:47:49,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:47:49,787 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:47:49,787 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:47:49,788 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:47:49,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:47:50,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:51,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:47:53,297 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:47:53,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:47:56,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:47:58,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:01,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:03,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:48:05,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:48:05,852 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:48:05,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:48:05,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:48:05,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:48:06,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:07,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:48:08,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:11,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:13,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:16,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:18,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:48:21,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:48:21,898 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:48:21,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:48:21,900 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:48:21,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:21,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:48:23,300 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:48:23,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:23,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:48:27,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:28,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:32,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:33,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:48:37,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:38,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:48:38,139 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:48:38,139 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:48:38,141 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:48:38,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:39,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:48:40,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:48:43,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:43,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:48,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:48,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:48:53,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:48:53,303 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:48:53,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:54,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:48:54,103 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:48:54,103 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:48:54,105 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:48:55,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:48:56,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:48:58,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:48:59,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:03,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:49:04,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:08,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:09,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:10,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:49:10,364 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:49:10,365 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:49:10,366 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:49:11,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:49:12,037 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:49:13,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:14,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:18,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:49:19,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:23,308 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:49:23,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:25,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:26,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:49:26,227 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:49:26,228 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:49:26,230 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:49:27,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:49:28,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:49:28,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:31,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:33,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:49:36,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:38,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:41,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:42,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:49:42,537 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:49:42,538 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:49:42,539 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:49:43,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:49:43,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:44,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:49:46,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:48,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:49:51,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:53,309 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:49:53,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:57,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:49:58,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:49:58,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:49:58,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:49:58,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:49:58,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:49:59,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:50:00,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:50:02,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:03,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:50:07,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:08,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:12,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:13,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:14,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:50:14,824 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:50:14,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:50:14,826 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:50:15,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:50:16,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:50:17,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:18,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:50:22,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:23,312 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:50:23,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:28,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:28,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:30,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:50:30,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:50:30,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:50:30,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:50:31,103 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:50:32,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:50:33,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:50:33,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:38,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:38,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:43,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:43,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:47,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:50:47,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:50:47,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:50:47,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:50:48,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:50:48,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:50:49,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:50,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:50:53,315 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:50:53,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:54,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:50:58,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:50:59,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:03,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:51:03,365 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:51:03,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:51:03,367 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:51:03,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:51:04,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:51:04,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:06,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:51:08,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:09,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:13,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:14,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:18,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:51:19,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:19,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:51:19,392 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:51:19,392 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:51:19,394 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:51:20,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:51:22,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:51:23,317 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:51:23,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:25,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:28,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:30,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:33,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:51:35,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:35,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:51:35,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:51:35,819 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:51:35,820 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:51:36,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:51:38,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:51:38,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:40,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:43,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:45,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:48,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:51:50,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:51,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:51:51,755 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:51:51,755 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:51:51,757 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:51:52,171 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:51:53,321 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:51:53,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:51:54,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:51:56,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:51:58,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:01,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:03,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:52:06,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:08,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:52:08,189 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:52:08,189 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:52:08,191 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:52:08,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:09,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:52:10,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:52:12,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:13,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:17,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:18,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:52:22,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:23,324 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:52:23,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:24,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:52:24,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:52:24,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:52:24,040 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:52:24,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:52:26,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:52:28,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:28,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:33,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:33,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:52:38,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:39,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:40,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:52:40,366 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:52:40,366 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:52:40,368 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:52:41,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:52:42,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:52:43,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:44,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:48,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:48,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:52:53,328 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:52:54,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:52:54,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:52:56,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:52:56,324 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:52:56,324 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:52:56,326 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:52:57,226 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:52:58,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:52:59,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:00,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:03,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:53:05,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:09,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:10,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:12,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:53:12,485 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:53:12,486 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:53:12,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:53:13,239 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:53:14,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:14,239 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:53:15,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:18,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:53:20,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:23,330 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:53:24,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:26,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:28,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:53:28,499 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:53:28,499 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:53:28,501 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:53:29,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:29,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:53:30,252 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:53:31,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:33,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:53:36,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:39,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:41,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:44,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:44,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:53:44,673 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:53:44,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:53:44,675 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:53:45,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:53:46,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:53:46,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:48,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:53:51,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:53,333 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:53:54,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:53:57,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:53:59,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:00,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:54:00,859 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:54:00,859 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:54:00,860 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:54:01,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:54:02,280 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:54:02,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:03,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:54:07,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:09,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:12,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:14,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:17,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:54:17,178 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:54:17,178 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:54:17,180 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:54:17,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:54:18,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:18,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:54:18,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:54:23,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:23,337 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:54:24,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:28,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:29,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:33,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:54:33,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:54:33,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:54:33,227 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:54:33,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:54:33,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:54:34,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:34,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:54:39,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:39,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:44,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:44,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:48,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:54:49,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:54:49,179 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:54:49,179 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:54:49,181 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:54:49,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:54:50,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:50,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:54:53,339 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:54:54,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:54:55,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:54:59,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:00,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:03,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:55:05,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:55:05,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:55:05,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:55:05,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:55:05,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:55:06,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:06,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:09,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:10,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:11,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:14,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:14,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:16,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:16,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:18,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:55:22,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:22,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:23,343 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:55:24,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:24,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:25,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:55:27,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:28,352 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:29,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:32,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:33,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:55:37,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:39,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:41,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:55:41,925 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:55:41,925 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:55:41,927 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:55:42,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:55:42,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:44,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:44,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:55:47,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:48,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:55:52,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:53,346 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:55:54,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:55:58,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:55:58,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:55:58,142 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:55:58,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:55:58,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:55:58,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:55:59,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:00,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:56:03,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:03,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:56:08,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:09,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:13,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:14,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:14,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:56:14,213 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:56:14,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:56:14,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:56:14,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:56:16,393 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:56:18,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:18,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:56:23,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:23,349 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:56:24,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:28,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:29,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:30,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:56:30,455 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:56:30,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:56:30,457 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:56:31,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:56:32,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:56:33,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:33,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:56:38,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:39,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:43,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:44,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:46,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:56:46,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:56:46,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:56:46,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:56:47,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:56:48,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:56:48,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:56:48,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:53,353 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:56:54,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:54,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:56:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:56:59,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:02,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:57:02,740 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:57:02,740 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:57:02,742 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:57:03,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:57:03,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:57:04,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:57:04,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:09,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:09,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:14,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:14,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:18,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:57:18,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:57:18,800 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:57:18,800 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:57:18,802 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:57:19,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:57:19,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:20,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:57:23,354 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:57:24,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:25,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:29,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:30,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:33,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:57:35,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:57:35,035 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:57:35,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:57:35,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:57:35,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:57:36,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:36,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:57:39,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:41,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:44,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:46,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:48,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:57:50,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:57:50,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:57:50,884 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:57:50,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:57:51,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:57:51,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:52,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:57:53,359 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:57:54,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:57:57,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:57:59,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:02,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:03,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:58:06,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:58:06,932 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:58:06,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:58:06,934 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:58:07,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:58:07,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:08,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:58:09,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:12,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:14,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:17,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:18,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:58:22,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:58:22,822 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:58:22,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:58:22,824 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:58:23,361 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:58:23,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:23,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:58:24,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:24,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:58:28,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:29,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:33,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:33,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:58:38,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:38,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:58:38,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:58:38,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:58:38,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:58:39,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:39,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:58:40,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:58:43,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:44,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:48,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:58:48,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:53,364 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:58:54,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:54,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:58:54,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:58:54,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:58:54,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:58:54,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:58:55,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:58:56,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:58:59,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:58:59,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:03,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:59:04,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:09,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:09,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:10,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:59:10,897 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:59:10,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:59:10,899 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:59:11,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:59:12,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:59:14,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:14,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:18,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:59:19,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:23,367 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:59:24,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:25,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:26,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:59:26,813 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:59:26,813 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:59:26,815 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:59:27,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:59:28,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:59:29,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:30,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:33,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:59:35,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:39,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:40,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:42,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:59:42,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:59:42,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:59:42,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:59:43,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 11:59:44,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:44,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 11:59:45,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:48,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 11:59:50,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:53,370 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 11:59:54,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:56,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 11:59:58,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 11:59:58,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 11:59:58,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 11:59:58,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 11:59:59,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 11:59:59,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:00:00,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:00:01,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:03,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:00:06,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:09,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:11,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:14,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:14,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:00:14,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:00:14,838 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:00:14,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:00:15,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:00:16,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:00:17,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:18,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:00:22,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:23,374 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:00:24,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:28,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:29,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:30,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:00:30,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:00:30,695 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:00:30,696 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:00:31,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:00:32,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:00:33,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:00:33,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:38,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:39,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:43,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:44,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:46,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:00:46,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:00:46,745 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:00:46,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:00:47,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:00:48,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:00:48,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:00:48,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:53,378 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:00:54,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:54,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:00:59,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:00:59,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:02,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:01:02,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:01:02,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:01:02,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:01:03,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:01:03,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:01:04,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:01:04,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:09,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:09,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:14,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:14,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:18,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:01:18,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:01:18,782 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:01:18,783 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:01:18,784 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:01:19,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:01:19,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:20,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:01:23,383 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:01:24,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:25,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:29,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:30,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:33,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:01:34,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:01:34,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:01:34,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:01:34,645 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:01:34,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:01:35,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:36,661 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:01:39,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:40,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:44,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:45,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:48,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:01:50,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:50,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:01:50,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:01:50,759 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:01:50,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:01:51,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:01:52,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:01:53,383 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:01:54,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:01:56,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:01:59,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:01,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:03,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:02:06,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:06,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:02:06,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:02:06,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:02:06,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:02:07,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:02:08,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:02:09,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:11,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:14,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:16,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:18,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:02:21,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:22,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:02:22,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:02:22,677 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:02:22,679 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:02:22,700 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:02:23,386 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:02:24,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:24,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:02:27,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:29,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:32,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:33,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:02:37,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:38,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:02:38,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:02:38,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:02:38,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:02:39,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:39,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:02:40,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:02:42,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:44,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:47,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:48,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:02:52,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:53,388 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:02:54,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:02:54,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:02:54,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:02:54,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:02:54,803 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:02:55,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:02:56,732 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:02:58,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:02:59,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:03,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:03:03,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:08,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:09,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:10,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:03:10,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:03:10,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:03:10,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:03:11,745 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:03:12,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:03:13,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:14,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:18,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:03:18,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:23,392 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:03:24,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:24,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:26,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:03:26,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:03:26,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:03:26,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:03:27,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:03:28,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:03:29,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:29,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:33,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:03:34,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:39,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:39,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:43,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:03:43,088 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:03:43,088 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:03:43,091 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:03:43,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:03:44,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:44,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:03:45,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:48,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:03:50,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:53,395 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:03:54,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:55,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:03:59,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:03:59,043 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:03:59,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:03:59,045 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:03:59,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:03:59,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:04:00,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:04:01,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:03,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:04:06,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:09,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:11,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:14,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:15,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:04:15,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:04:15,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:04:15,309 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:04:15,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:04:16,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:16,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:04:18,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:04:21,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:23,398 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:04:24,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:26,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:29,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:31,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:04:31,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:04:31,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:04:31,166 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:04:31,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:04:32,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:32,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:04:33,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:04:37,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:39,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:42,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:44,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:47,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:47,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:04:47,514 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:04:47,515 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:04:47,517 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:04:47,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:04:48,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:04:48,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:04:52,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:53,401 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:04:54,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:04:58,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:04:59,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:03,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:03,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:05:03,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:05:03,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:05:03,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:05:03,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:05:03,838 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:05:04,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:05:08,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:09,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:13,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:14,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:18,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:18,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:05:19,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:05:19,652 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:05:19,652 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:05:19,655 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:05:19,851 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:05:20,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:05:23,404 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:05:24,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:24,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:29,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:29,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:33,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:05:34,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:35,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:05:35,699 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:05:35,699 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:05:35,701 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:05:35,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:05:36,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:05:39,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:39,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:44,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:44,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:48,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:05:49,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:51,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:05:51,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:05:51,942 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:05:51,944 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:05:52,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:05:52,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:05:53,407 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:05:54,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:05:55,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:05:59,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:00,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:03,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:06:05,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:08,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:06:08,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:06:08,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:06:08,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:06:08,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:06:08,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:06:09,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:11,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:14,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:16,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:18,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:06:21,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:23,412 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:06:24,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:06:24,116 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:06:24,117 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:06:24,118 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:06:24,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:24,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:06:26,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:26,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:06:29,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:31,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:33,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:06:36,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:39,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:40,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:06:40,379 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:06:40,379 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:06:40,380 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:06:40,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:06:41,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:42,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:06:44,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:46,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:48,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:06:51,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:53,414 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:06:54,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:06:56,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:06:56,308 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:06:56,309 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:06:56,310 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:06:56,938 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:06:57,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:06:58,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:06:59,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:02,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:03,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:07:07,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:09,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:12,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:12,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:07:12,428 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:07:12,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:07:12,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:07:12,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:07:14,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:14,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:07:17,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:18,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:07:22,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:23,417 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:07:24,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:28,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:07:28,364 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:07:28,364 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:07:28,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:28,366 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:07:28,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:07:29,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:30,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:07:33,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:33,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:07:38,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:39,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:43,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:44,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:07:44,501 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:07:44,502 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:07:44,504 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:07:44,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:44,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:07:46,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:07:48,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:48,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:07:53,422 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:07:54,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:54,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:07:59,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:07:59,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:00,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:08:00,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:08:00,358 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:08:00,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:08:00,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:08:02,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:08:03,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:08:05,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:09,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:10,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:14,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:15,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:16,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:08:16,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:08:16,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:08:16,631 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:08:17,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:08:18,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:08:19,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:08:20,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:23,422 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:08:24,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:26,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:29,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:31,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:32,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:08:32,481 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:08:32,482 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:08:32,483 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:08:33,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:08:33,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:08:35,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:08:36,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:39,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:41,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:44,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:46,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:48,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:08:48,607 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:08:48,608 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:08:48,609 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:08:48,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:08:49,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:08:51,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:08:51,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:53,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:08:54,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:08:57,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:08:59,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:02,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:03,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:09:04,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:09:04,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:09:04,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:09:04,463 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:09:05,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:09:07,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:09:07,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:09,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:12,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:14,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:17,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:18,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:09:20,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:09:20,714 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:09:20,715 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:09:20,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:09:21,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:09:22,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:23,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:09:23,429 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:09:24,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:28,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:29,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:33,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:33,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:09:36,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:09:36,729 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:09:36,730 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:09:36,731 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:09:37,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:09:38,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:39,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:09:39,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:43,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:44,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:48,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:09:48,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:52,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:09:52,680 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:09:52,680 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:09:52,682 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:09:53,088 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:09:53,444 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:09:54,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:54,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:09:55,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:09:59,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:09:59,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:03,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:10:04,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:08,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:10:08,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:10:08,748 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:10:08,750 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:10:09,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:10:09,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:09,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:11,103 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:10:14,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:14,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:18,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:10:19,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:23,435 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:10:24,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:24,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:10:24,795 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:10:24,795 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:10:24,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:10:24,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:25,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:10:27,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:10:29,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:29,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:33,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:10:34,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:39,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:39,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:40,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:10:40,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:10:40,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:10:40,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:10:41,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:10:43,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:10:44,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:45,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:48,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:10:50,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:53,438 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:10:54,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:10:56,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:10:56,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:10:56,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:10:56,806 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:10:56,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:10:57,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:10:59,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:10:59,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:01,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:03,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:11:06,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:09,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:11,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:12,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:11:12,650 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:11:12,651 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:11:12,652 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:11:13,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:11:14,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:15,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:11:17,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:18,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:11:22,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:23,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:11:24,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:28,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:28,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:11:28,707 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:11:28,708 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:11:28,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:11:29,171 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:11:29,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:31,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:11:33,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:11:33,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:38,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:39,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:43,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:44,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:44,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:11:44,774 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:11:44,774 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:11:44,776 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:11:45,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:11:47,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:11:48,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:11:48,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:53,444 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:11:54,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:54,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:11:59,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:11:59,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:00,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:12:00,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:12:00,806 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:12:00,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:12:01,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:12:03,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:12:03,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:12:04,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:09,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:09,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:14,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:14,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:16,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:12:16,855 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:12:16,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:12:16,857 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:12:17,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:12:18,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:12:19,213 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:12:19,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:23,448 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:12:24,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:25,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:29,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:30,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:32,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:12:32,786 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:12:32,786 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:12:32,789 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:12:33,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:12:33,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:12:35,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:12:35,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:39,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:40,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:44,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:45,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:48,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:12:48,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:12:48,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:12:48,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:12:48,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:12:49,241 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:12:50,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:51,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:12:53,450 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:12:54,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:12:56,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:12:59,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:01,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:03,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:13:04,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:13:04,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:13:04,981 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:13:04,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:13:05,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:13:07,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:07,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:13:09,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:12,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:14,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:17,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:18,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:13:21,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:13:21,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:13:21,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:13:21,244 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:13:21,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:13:22,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:23,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:13:23,453 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:13:24,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:27,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:29,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:33,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:13:37,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:13:37,199 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:13:37,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:13:37,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:13:37,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:13:38,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:39,286 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:13:39,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:43,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:44,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:48,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:48,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:13:53,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:53,457 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:13:53,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:13:53,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:13:53,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:13:53,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:13:54,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:13:54,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:13:55,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:13:58,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:13:59,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:03,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:03,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:14:08,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:09,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:09,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:14:09,640 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:14:09,640 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:14:09,642 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:14:10,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:14:11,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:14:13,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:14,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:18,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:14:18,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:23,459 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:14:24,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:24,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:25,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:14:25,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:14:25,858 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:14:25,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:14:26,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:14:27,328 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:14:29,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:29,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:33,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:14:34,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:39,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:39,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:41,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:14:41,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:14:41,789 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:14:41,791 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:14:42,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:14:43,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:14:44,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:45,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:48,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:14:50,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:53,464 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:14:53,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:56,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:14:58,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:14:58,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:14:58,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:14:58,166 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:14:58,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:14:58,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:14:59,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:15:02,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:03,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:15:07,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:08,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:12,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:13,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:14,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:15:14,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:15:14,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:15:14,104 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:15:14,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:15:15,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:15:18,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:18,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:15:23,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:23,472 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:15:23,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:28,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:28,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:30,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:15:30,241 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:15:30,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:15:30,243 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:15:30,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:15:31,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:15:33,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:15:34,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:38,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:39,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:43,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:44,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:46,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:15:46,410 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:15:46,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:15:46,413 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:15:47,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:15:47,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:15:48,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:15:49,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:53,469 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:15:53,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:54,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:15:58,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:15:59,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:02,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:16:02,544 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:16:02,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:16:02,547 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:16:03,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:16:03,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:16:03,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:16:04,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:08,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:09,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:13,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:14,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:18,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:16:18,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:16:18,749 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:16:18,750 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:16:18,752 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:16:19,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:16:19,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:16:19,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:23,472 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:16:23,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:25,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:28,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:30,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:33,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:16:34,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:16:34,680 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:16:34,680 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:16:34,681 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:16:35,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:16:35,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:16:35,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:38,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:40,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:43,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:45,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:48,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:16:50,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:51,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:16:51,011 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:16:51,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:16:51,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:16:51,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:16:53,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:16:53,475 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:16:53,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:16:56,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:16:58,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:01,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:03,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:17:06,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:07,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:17:07,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:17:07,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:17:07,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:17:07,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:17:08,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:09,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:17:12,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:13,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:17,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:18,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:17:22,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:23,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:17:23,139 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:17:23,140 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:17:23,143 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:17:23,478 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:17:23,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:17:23,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:25,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:17:27,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:28,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:32,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:33,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:17:37,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:38,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:39,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:17:39,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:17:39,169 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:17:39,170 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:17:39,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:17:41,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:17:43,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:43,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:48,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:48,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:17:53,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:53,481 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:17:53,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:17:55,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:17:55,314 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:17:55,315 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:17:55,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:17:55,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:17:57,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:17:58,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:17:58,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:03,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:03,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:18:08,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:08,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:11,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:18:11,156 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:18:11,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:18:11,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:18:11,522 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:18:13,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:18:13,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:14,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:18,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:18:19,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:23,486 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:18:23,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:24,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:27,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:18:27,404 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:18:27,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:18:27,406 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:18:27,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:18:28,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:29,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:18:30,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:33,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:18:35,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:38,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:40,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:43,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:18:43,257 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:18:43,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:18:43,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:18:43,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:18:43,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:45,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:18:46,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:48,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:18:51,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:53,488 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:18:53,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:56,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:18:58,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:18:59,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:18:59,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:18:59,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:18:59,423 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:18:59,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:19:01,565 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:19:02,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:03,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:19:07,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:08,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:12,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:13,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:15,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:19:15,290 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:19:15,291 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:19:15,292 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:19:15,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:19:17,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:19:18,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:18,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:19:23,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:23,490 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:19:23,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:28,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:28,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:31,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:19:31,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:19:31,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:19:31,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:19:31,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:19:33,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:19:33,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:19:34,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:38,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:39,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:43,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:44,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:47,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:19:47,371 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:19:47,371 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:19:47,373 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:19:47,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:19:48,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:19:49,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:19:50,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:53,494 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:19:53,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:19:55,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:19:58,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:00,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:03,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:20:03,429 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:20:03,430 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:20:03,431 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:20:03,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:20:03,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:20:05,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:20:06,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:08,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:11,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:13,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:16,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:18,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:20:19,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:20:19,396 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:20:19,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:20:19,398 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:20:19,634 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:20:21,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:21,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:20:23,496 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:20:23,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:26,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:28,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:31,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:33,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:20:35,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:20:35,441 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:20:35,441 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:20:35,443 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:20:35,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:20:37,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:37,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:20:38,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:42,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:43,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:47,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:48,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:20:51,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:20:51,287 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:20:51,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:20:51,289 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:20:51,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:20:53,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:53,498 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:20:53,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:20:53,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:20:58,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:20:58,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:03,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:03,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:21:07,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:21:07,436 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:21:07,436 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:21:07,438 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:21:07,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:21:08,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:09,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:09,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:21:13,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:14,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:18,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:21:19,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:23,502 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:21:23,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:21:23,518 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:21:23,518 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:21:23,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:21:23,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:21:23,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:24,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:25,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:21:28,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:29,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:33,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:21:34,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:38,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:39,444 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:21:39,447 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:21:39,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:21:39,448 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:21:39,705 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:21:40,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:41,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:21:43,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:45,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:48,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:21:50,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:53,506 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:21:53,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:21:55,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:21:55,408 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:21:55,409 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:21:55,410 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:21:55,718 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:21:56,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:21:57,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:21:58,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:01,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:03,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:22:06,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:08,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:11,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:11,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:22:11,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:22:11,559 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:22:11,560 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:22:11,732 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:22:13,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:22:13,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:16,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:18,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:22:21,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:23,508 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:22:23,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:27,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:27,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:22:27,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:22:27,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:22:27,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:22:27,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:22:28,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:29,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:22:32,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:33,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:22:37,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:38,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:42,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:43,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:22:43,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:22:43,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:22:43,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:22:43,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:22:43,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:45,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:22:48,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:48,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:22:53,511 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:22:53,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:54,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:58,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:22:59,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:22:59,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:22:59,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:22:59,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:22:59,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:23:00,776 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:23:01,776 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:23:03,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:23:04,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:08,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:09,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:13,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:14,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:15,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:23:15,767 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:23:15,767 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:23:15,769 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:23:15,788 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:23:17,789 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:23:18,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:23:20,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:23,515 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:23:23,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:26,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:28,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:31,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:31,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:23:31,989 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:23:31,989 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:23:31,991 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:23:32,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:23:33,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:23:33,803 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:23:36,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:38,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:41,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:43,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:46,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:47,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:23:47,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:23:47,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:23:47,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:23:48,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:23:48,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:23:49,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:23:52,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:53,517 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:23:53,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:23:58,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:23:58,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:03,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:03,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:24:04,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:24:04,278 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:24:04,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:24:04,280 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:24:04,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:24:05,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:24:08,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:09,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:13,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:14,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:18,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:24:19,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:20,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:24:20,434 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:24:20,434 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:24:20,436 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:24:20,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:24:21,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:24:23,520 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:24:23,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:24,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:28,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:29,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:33,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:24:34,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:36,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:24:36,569 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:24:36,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:24:36,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:24:36,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:24:37,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:24:38,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:39,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:43,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:44,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:48,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:24:49,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:52,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:24:52,609 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:24:52,609 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:24:52,611 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:24:52,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:24:53,524 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:24:53,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:24:53,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:24:55,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:24:58,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:00,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:03,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:25:05,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:08,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:08,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:25:08,948 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:25:08,948 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:25:08,950 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:25:09,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:25:09,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:25:10,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:13,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:15,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:18,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:25:20,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:23,526 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:25:23,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:24,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:25:24,902 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:25:24,903 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:25:24,904 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:25:25,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:25:25,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:25:26,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:28,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:31,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:33,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:25:36,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:38,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:41,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:25:41,133 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:25:41,134 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:25:41,135 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:25:41,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:25:41,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:25:42,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:43,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:47,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:48,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:25:52,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:53,528 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:25:53,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:25:57,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:25:57,055 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:25:57,055 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:25:57,057 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:25:57,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:25:57,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:25:58,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:25:58,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:03,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:03,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:26:08,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:08,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:13,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:13,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:26:13,366 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:26:13,367 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:26:13,368 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:26:13,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:26:13,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:26:13,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:18,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:18,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:26:23,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:23,532 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:26:23,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:28,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:28,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:29,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:26:29,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:26:29,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:26:29,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:26:29,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:26:31,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:26:33,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:33,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:26:38,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:38,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:43,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:43,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:45,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:26:45,686 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:26:45,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:26:45,688 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:26:45,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:26:47,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:26:48,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:26:48,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:53,538 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:26:53,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:54,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:26:58,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:26:59,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:01,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:27:01,906 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:27:01,906 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:27:01,908 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:27:01,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:27:03,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:27:03,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:27:04,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:08,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:09,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:13,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:14,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:17,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:27:17,756 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:27:17,756 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:27:17,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:27:17,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:27:18,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:27:19,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:27:20,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:23,538 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:27:23,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:26,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:28,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:31,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:33,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:27:33,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:27:33,902 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:27:33,902 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:27:33,903 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:27:33,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:27:35,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:27:36,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:38,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:41,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:43,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:46,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:48,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:27:49,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:27:49,750 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:27:49,751 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:27:49,752 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:27:50,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:27:52,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:27:52,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:53,541 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:27:53,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:27:58,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:27:59,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:03,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:03,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:28:05,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:28:05,915 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:28:05,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:28:05,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:28:06,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:28:08,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:28:08,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:09,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:13,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:14,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:18,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:28:18,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:21,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:28:21,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:28:21,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:28:21,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:28:22,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:28:23,545 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:28:24,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:28:24,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:24,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:29,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:29,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:33,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:28:34,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:38,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:28:38,036 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:28:38,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:28:38,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:28:38,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:28:39,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:40,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:28:40,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:44,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:45,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:48,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:28:50,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:53,547 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:28:53,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:28:53,893 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:28:53,894 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:28:53,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:28:54,058 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:28:54,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:28:55,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:28:56,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:28:59,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:00,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:03,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:29:05,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:09,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:10,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:29:10,130 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:29:10,131 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:29:10,132 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:29:11,072 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:29:11,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:12,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:29:14,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:16,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:18,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:29:21,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:23,550 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:29:24,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:25,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:29:25,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:29:25,991 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:29:25,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:29:26,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:29:26,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:28,085 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:29:29,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:31,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:33,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:29:36,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:39,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:41,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:42,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:29:42,224 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:29:42,225 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:29:42,226 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:29:43,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:29:44,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:29:44,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:47,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:48,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:29:52,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:53,554 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:29:54,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:29:57,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:29:58,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:29:58,471 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:29:58,472 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:29:58,473 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:29:59,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:29:59,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:00,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:30:03,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:03,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:30:08,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:09,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:13,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:14,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:14,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:30:14,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:30:14,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:30:14,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:30:15,122 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:30:16,123 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:30:18,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:30:19,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:23,558 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:30:24,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:24,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:29,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:29,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:30,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:30:30,346 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:30:30,346 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:30:30,347 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:30:31,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:30:32,136 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:30:33,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:30:35,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:39,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:40,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:44,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:45,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:46,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:30:46,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:30:46,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:30:46,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:30:47,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:30:48,149 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:30:48,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:30:50,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:53,560 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:30:54,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:30:55,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:30:59,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:00,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:02,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:31:02,437 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:31:02,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:31:02,439 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:31:03,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:31:03,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:31:04,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:31:06,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:09,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:11,440 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:14,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:16,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:18,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:31:18,394 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:31:18,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:31:18,396 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:31:18,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:31:19,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:31:20,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:31:22,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:23,563 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:31:24,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:27,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:29,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:32,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:33,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:31:34,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:31:34,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:31:34,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:31:34,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:31:35,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:31:36,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:31:37,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:39,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:42,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:44,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:47,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:48,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:31:50,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:31:50,506 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:31:50,506 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:31:50,508 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:31:51,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:31:52,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:31:53,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:53,568 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:31:54,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:31:58,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:31:59,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:03,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:03,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:32:06,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:32:06,580 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:32:06,580 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:32:06,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:32:07,219 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:32:08,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:32:08,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:09,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:13,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:14,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:18,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:18,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:32:22,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:32:22,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:32:22,727 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:32:22,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:32:23,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:32:23,569 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:32:24,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:32:24,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:24,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:29,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:29,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:33,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:32:34,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:38,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:32:38,866 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:32:38,866 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:32:38,867 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:32:39,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:39,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:32:39,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:40,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:32:44,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:44,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:48,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:32:49,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:53,572 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:32:54,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:54,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:32:54,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:32:54,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:32:54,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:32:54,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:32:55,254 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:32:56,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:32:59,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:32:59,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:03,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:33:04,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:09,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:09,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:10,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:33:10,934 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:33:10,934 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:33:10,936 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:33:11,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:33:12,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:33:14,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:15,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:18,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:33:20,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:23,577 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:33:24,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:26,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:27,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:33:27,082 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:33:27,082 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:33:27,084 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:33:27,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:33:28,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:33:29,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:32,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:33,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:33:37,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:39,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:42,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:43,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:33:43,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:33:43,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:33:43,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:33:43,288 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:33:44,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:44,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:33:47,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:48,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:33:52,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:53,580 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:33:54,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:57,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:33:59,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:33:59,202 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:33:59,203 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:33:59,204 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:33:59,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:33:59,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:34:00,301 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:34:03,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:03,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:34:08,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:09,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:13,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:14,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:15,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:34:15,604 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:34:15,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:34:15,606 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:34:16,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:34:16,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:34:18,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:18,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:34:23,585 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:34:24,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:24,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:29,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:29,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:31,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:34:31,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:34:31,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:34:31,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:34:32,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:34:32,325 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:34:33,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:34:35,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:39,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:40,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:44,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:45,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:47,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:34:47,679 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:34:47,680 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:34:47,681 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:34:48,336 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:34:48,337 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:34:48,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:34:50,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:53,588 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:34:54,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:34:56,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:34:59,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:01,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:03,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:35:03,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:35:03,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:35:03,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:35:03,702 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:35:04,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:35:04,349 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:35:06,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:09,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:11,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:14,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:16,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:18,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:35:19,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:35:19,978 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:35:19,979 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:35:19,980 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:35:20,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:35:20,364 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:35:22,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:23,593 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:35:24,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:27,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:29,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:32,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:33,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:35:35,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:35:35,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:35:35,921 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:35:35,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:35:36,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:35:36,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:35:37,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:39,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:42,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:44,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:47,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:48,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:35:51,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:35:51,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:35:51,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:35:51,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:35:52,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:35:52,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:35:53,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:53,595 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:35:54,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:35:58,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:35:59,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:03,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:03,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:36:07,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:36:07,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:36:07,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:36:07,826 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:36:08,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:36:08,401 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:36:08,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:09,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:13,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:14,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:18,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:36:18,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:23,598 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:36:24,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:36:24,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:36:24,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:36:24,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:24,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:36:24,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:24,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:36:26,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:36:29,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:29,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:33,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:36:34,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:39,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:39,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:40,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:36:40,049 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:36:40,049 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:36:40,051 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:36:40,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:36:40,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:36:44,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:45,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:48,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:36:50,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:53,600 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:36:54,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:36:55,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:36:56,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:36:56,020 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:36:56,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:36:56,022 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:36:56,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:36:56,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:36:59,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:01,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:03,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:37:06,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:09,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:11,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:12,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:37:12,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:37:12,452 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:37:12,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:37:12,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:37:14,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:14,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:37:16,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:18,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:37:21,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:23,603 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:37:24,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:26,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:28,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:37:28,428 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:37:28,428 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:37:28,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:37:28,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:37:29,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:30,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:37:32,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:33,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:37:37,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:39,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:42,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:44,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:44,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:37:44,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:37:44,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:37:44,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:37:44,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:37:46,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:37:47,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:48,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:37:52,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:53,606 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:37:54,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:37:57,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:37:59,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:00,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:38:00,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:38:00,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:38:00,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:38:00,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:38:02,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:38:03,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:03,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:38:08,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:09,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:13,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:14,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:16,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:38:16,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:38:16,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:38:16,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:38:16,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:38:18,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:18,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:38:18,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:38:23,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:23,609 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:38:24,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:28,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:29,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:32,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:38:32,416 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:38:32,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:38:32,417 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:38:32,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:38:33,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:38:34,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:34,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:38:39,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:39,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:44,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:44,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:48,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:38:48,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:38:48,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:38:48,581 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:38:48,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:38:49,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:38:49,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:50,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:38:53,612 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:38:54,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:54,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:38:59,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:38:59,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:03,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:39:04,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:39:04,431 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:39:04,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:39:04,433 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:39:04,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:39:05,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:06,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:39:09,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:10,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:14,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:15,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:18,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:39:20,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:20,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:39:20,739 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:39:20,739 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:39:20,740 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:39:21,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:39:22,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:39:23,615 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:39:24,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:25,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:29,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:30,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:33,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:39:35,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:36,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:39:36,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:39:36,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:39:36,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:39:37,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:39:38,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:39:39,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:40,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:44,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:45,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:48,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:39:50,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:52,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:39:52,844 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:39:52,845 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:39:52,846 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:39:53,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:39:53,619 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:39:54,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:39:54,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:39:56,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:39:59,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:01,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:03,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:40:06,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:09,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:40:09,066 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:40:09,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:40:09,068 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:40:09,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:09,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:40:10,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:40:12,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:14,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:17,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:18,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:40:22,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:23,623 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:40:24,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:24,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:40:24,911 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:40:24,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:40:24,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:40:25,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:40:26,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:40:27,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:29,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:32,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:33,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:40:37,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:39,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:41,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:40:41,057 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:40:41,057 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:40:41,059 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:40:41,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:40:42,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:40:43,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:44,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:48,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:48,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:40:53,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:53,625 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:40:54,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:40:57,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:40:57,196 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:40:57,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:40:57,198 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:40:57,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:40:58,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:40:58,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:40:59,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:03,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:03,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:41:08,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:09,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:13,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:13,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:41:13,232 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:41:13,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:41:13,234 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:41:13,646 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:41:14,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:14,646 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:41:18,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:18,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:41:23,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:23,628 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:41:24,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:28,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:29,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:41:29,177 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:41:29,177 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:41:29,179 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:41:29,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:29,658 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:41:30,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:41:33,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:41:34,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:39,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:39,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:44,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:44,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:45,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:41:45,330 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:41:45,331 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:41:45,332 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:41:45,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:41:46,672 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:41:48,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:41:49,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:53,631 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:41:54,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:54,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:41:59,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:41:59,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:01,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:42:01,276 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:42:01,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:42:01,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:42:01,684 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:42:02,685 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:42:03,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:42:05,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:09,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:10,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:14,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:15,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:17,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:42:17,304 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:42:17,305 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:42:17,306 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:42:17,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:42:18,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:42:18,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:42:20,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:23,633 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:42:24,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:25,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:29,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:30,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:33,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:42:33,344 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:42:33,345 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:42:33,346 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:42:33,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:42:33,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:42:34,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:42:36,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:39,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:41,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:44,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:46,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:48,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:42:49,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:42:49,484 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:42:49,484 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:42:49,486 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:42:49,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:42:50,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:42:51,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:53,636 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:42:54,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:42:56,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:42:59,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:01,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:03,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:43:05,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:43:05,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:43:05,670 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:43:05,671 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:43:05,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:43:06,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:06,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:43:09,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:11,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:14,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:16,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:18,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:43:21,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:43:21,604 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:43:21,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:43:21,606 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:43:21,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:43:22,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:22,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:43:23,639 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:43:24,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:28,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:29,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:33,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:43:33,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:37,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:43:37,858 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:43:37,859 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:43:37,860 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:43:38,761 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:43:38,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:43:38,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:39,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:43,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:44,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:48,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:43:48,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:53,644 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:43:53,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:43:53,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:43:53,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:43:53,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:43:53,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:54,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:43:54,773 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:43:54,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:43:58,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:43:59,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:03,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:44:03,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:08,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:09,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:09,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:44:09,820 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:44:09,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:44:09,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:44:10,787 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:44:10,787 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:44:14,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:14,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:18,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:44:19,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:23,646 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:44:24,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:25,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:26,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:44:26,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:44:26,244 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:44:26,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:44:26,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:44:26,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:44:29,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:31,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:33,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:44:36,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:39,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:41,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:42,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:44:42,188 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:44:42,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:44:42,190 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:44:42,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:44:42,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:44:44,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:47,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:48,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:44:52,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:53,649 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:44:54,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:44:57,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:44:58,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:44:58,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:44:58,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:44:58,320 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:44:58,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:44:58,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:44:59,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:03,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:03,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:45:08,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:09,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:13,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:14,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:14,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:45:14,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:45:14,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:45:14,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:45:14,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:45:14,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:45:18,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:18,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:45:23,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:23,653 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:45:24,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:28,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:29,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:30,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:45:30,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:45:30,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:45:30,629 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:45:30,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:45:30,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:45:33,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:45:34,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:39,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:39,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:44,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:44,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:46,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:45:46,584 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:45:46,584 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:45:46,586 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:45:46,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:45:46,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:45:48,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:45:50,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:53,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:45:54,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:45:55,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:45:59,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:00,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:02,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:46:02,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:46:02,606 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:46:02,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:46:02,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:46:02,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:46:03,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:46:06,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:09,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:11,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:14,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:16,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:18,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:46:18,463 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:46:18,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:46:18,465 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:46:18,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:46:18,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:46:18,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:46:22,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:23,658 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:46:24,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:27,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:29,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:32,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:33,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:46:34,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:46:34,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:46:34,699 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:46:34,700 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:46:34,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:46:35,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:46:37,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:39,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:42,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:44,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:47,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:48,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:46:50,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:46:50,642 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:46:50,642 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:46:50,643 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:46:50,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:46:51,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:46:53,661 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:46:53,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:54,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:46:58,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:46:59,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:03,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:47:03,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:06,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:47:06,571 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:47:06,571 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:47:06,574 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:47:06,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:47:07,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:47:09,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:09,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:14,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:14,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:18,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:47:19,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:23,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:47:23,043 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:47:23,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:47:23,045 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:47:23,664 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:47:23,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:47:24,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:24,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:25,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:47:29,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:29,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:33,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:47:34,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:39,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:47:39,013 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:47:39,014 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:47:39,015 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:47:39,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:39,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:47:40,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:41,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:47:44,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:45,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:48,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:47:50,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:53,667 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:47:54,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:47:55,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:47:55,058 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:47:55,058 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:47:55,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:47:55,060 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:47:55,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:47:57,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:47:59,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:00,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:03,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:48:05,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:09,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:10,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:10,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:48:10,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:48:10,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:48:10,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:48:10,980 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:48:13,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:48:14,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:15,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:18,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:48:20,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:23,670 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:48:24,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:26,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:27,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:48:27,119 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:48:27,120 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:48:27,121 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:48:27,993 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:48:29,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:29,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:48:32,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:33,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:48:37,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:39,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:42,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:43,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:48:43,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:48:43,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:48:43,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:48:44,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:48:44,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:46,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:48:48,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:48,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:48:53,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:53,674 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:48:54,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:48:58,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:48:59,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:48:59,250 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:48:59,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:48:59,252 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:48:59,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:00,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:49:02,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:03,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:49:04,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:04,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:08,024 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:09,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:10,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:10,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:14,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:16,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:16,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:18,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:18,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:49:18,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:49:20,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:21,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:23,676 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:49:23,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:26,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:28,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:31,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:33,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:49:34,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:49:34,741 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:49:34,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:49:34,742 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:49:35,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:49:36,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:36,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:38,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:41,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:43,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:46,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:48,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:49:50,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:49:50,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:49:50,954 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:49:50,955 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:49:51,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:49:51,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:52,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:49:53,680 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:49:53,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:49:57,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:49:58,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:02,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:03,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:50:07,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:50:07,117 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:50:07,117 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:50:07,119 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:50:08,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:50:08,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:08,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:10,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:50:13,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:13,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:18,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:18,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:50:23,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:23,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:50:23,294 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:50:23,294 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:50:23,295 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:50:23,684 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:50:23,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:24,089 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:50:26,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:50:28,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:28,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:33,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:50:33,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:38,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:38,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:39,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:50:39,331 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:50:39,332 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:50:39,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:50:40,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:50:42,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:50:43,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:44,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:48,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:50:49,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:53,687 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:50:53,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:50:54,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:50:55,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:50:55,386 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:50:55,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:50:55,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:50:56,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:50:58,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:50:58,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:00,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:03,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:51:05,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:08,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:10,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:11,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:51:11,335 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:51:11,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:51:11,337 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:51:12,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:51:13,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:14,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:51:16,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:18,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:51:21,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:23,689 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:51:23,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:26,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:27,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:51:27,293 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:51:27,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:51:27,294 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:51:28,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:51:28,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:30,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:51:32,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:33,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:51:37,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:38,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:42,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:43,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:51:43,249 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:51:43,250 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:51:43,251 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:51:43,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:44,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:51:46,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:51:48,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:48,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:51:53,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:53,692 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:51:53,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:58,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:51:58,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:51:59,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:51:59,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:51:59,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:51:59,124 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:51:59,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:52:00,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:52:03,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:52:04,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:08,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:09,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:13,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:14,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:15,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:52:15,191 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:52:15,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:52:15,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:52:16,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:52:18,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:52:18,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:52:19,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:23,695 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:52:23,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:24,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:28,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:29,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:31,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:52:31,144 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:52:31,145 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:52:31,146 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:52:31,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:52:32,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:52:33,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:52:35,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:38,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:40,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:43,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:45,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:47,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:52:47,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:52:47,706 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:52:47,708 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:52:48,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:52:48,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:52:50,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:52:50,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:53,698 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:52:53,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:52:56,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:52:58,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:01,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:03,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:53:03,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:53:03,765 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:53:03,765 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:53:03,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:53:04,221 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:53:06,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:53:06,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:08,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:11,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:13,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:16,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:18,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:53:19,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:53:19,722 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:53:19,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:53:19,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:53:20,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:53:22,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:53:22,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:23,701 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:53:23,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:28,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:28,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:33,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:53:33,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:35,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:53:35,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:53:35,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:53:35,695 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:53:36,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:53:38,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:53:38,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:39,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:43,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:44,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:48,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:53:48,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:51,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:53:51,939 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:53:51,940 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:53:51,941 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:53:52,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:53:53,705 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:53:54,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:54,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:53:54,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:53:59,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:53:59,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:03,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:54:04,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:07,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:54:07,817 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:54:07,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:54:07,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:54:08,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:54:09,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:09,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:10,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:54:14,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:14,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:18,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:54:19,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:23,707 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:54:23,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:54:23,877 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:54:23,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:54:23,879 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:54:24,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:24,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:54:24,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:26,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:54:29,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:29,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:33,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:54:34,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:39,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:39,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:39,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:54:39,958 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:54:39,958 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:54:39,960 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:54:40,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:54:42,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:54:44,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:44,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:48,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:54:49,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:53,710 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:54:54,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:54:55,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:54:56,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:54:56,115 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:54:56,115 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:54:56,117 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:54:56,315 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:54:58,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:54:59,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:01,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:03,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:55:06,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:09,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:11,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:12,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:55:12,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:55:12,072 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:55:12,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:55:12,329 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:55:14,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:14,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:55:17,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:18,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:55:22,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:23,713 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:55:24,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:27,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:27,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:55:27,920 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:55:27,920 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:55:27,922 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:55:28,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:55:29,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:30,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:55:32,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:33,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:55:37,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:39,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:42,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:44,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:44,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:55:44,178 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:55:44,179 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:55:44,180 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:55:44,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:55:46,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:55:48,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:48,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:55:53,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:53,716 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:55:54,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:55:58,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:55:59,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:00,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:56:00,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:56:00,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:56:00,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:56:00,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:56:02,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:56:03,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:56:04,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:09,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:09,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:14,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:14,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:16,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:56:16,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:56:16,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:56:16,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:56:16,380 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:56:18,381 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:56:18,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:56:19,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:23,719 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:56:24,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:24,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:29,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:29,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:32,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:56:32,316 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:56:32,316 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:56:32,318 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:56:32,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:56:33,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:56:34,394 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:56:35,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:39,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:40,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:44,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:45,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:48,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:56:48,386 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:56:48,386 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:56:48,388 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:56:48,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:56:48,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:56:50,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:50,407 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:56:53,722 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:56:54,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:56:55,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:56:59,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:00,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:03,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:57:04,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:57:04,240 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:57:04,240 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:57:04,242 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:57:04,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:57:06,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:06,421 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:57:09,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:11,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:14,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:16,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:18,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:57:20,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:57:20,582 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:57:20,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:57:20,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:57:21,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:57:21,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:22,434 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:57:23,727 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:57:24,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:26,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:29,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:31,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:33,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:57:36,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:57:36,566 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:57:36,566 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:57:36,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:57:37,446 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:57:37,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:38,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:57:39,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:42,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:44,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:47,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:48,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:57:52,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:52,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:57:52,691 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:57:52,692 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:57:52,693 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:57:53,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:57:53,729 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:57:54,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:57:54,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:57:57,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:57:59,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:02,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:03,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:58:07,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:08,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:58:08,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:58:08,874 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:58:08,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:58:09,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:09,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:58:10,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:58:12,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:14,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:17,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:18,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:58:22,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:23,732 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:58:24,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:25,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:58:25,120 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:58:25,120 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:58:25,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:58:25,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:58:26,485 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:58:28,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:29,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:33,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:33,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:58:38,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:39,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:41,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:58:41,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:58:41,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:58:41,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:58:41,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:58:42,498 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:58:43,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:44,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:48,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:48,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:58:53,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:53,736 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:58:54,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:58:57,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:58:57,520 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:58:57,521 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:58:57,522 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:58:58,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:58:58,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:58:58,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:58:59,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:03,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:03,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:59:08,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:09,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:13,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:13,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:59:13,725 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:59:13,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:59:13,727 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:59:14,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:14,527 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:59:16,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:59:18,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:59:18,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:23,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:23,737 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:59:24,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:28,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:29,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:29,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:59:29,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:59:29,862 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:59:29,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:59:30,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:59:32,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:59:33,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:59:33,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:38,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:39,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:43,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:44,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:46,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 12:59:46,146 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 12:59:46,146 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 12:59:46,147 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 12:59:46,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 12:59:48,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 12:59:48,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 12:59:49,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:53,740 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 12:59:54,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:54,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 12:59:59,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 12:59:59,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:02,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:00:02,278 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:00:02,278 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:00:02,280 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:00:02,566 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:00:03,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:00:04,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:00:05,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:09,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:10,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:14,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:15,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:18,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:00:18,398 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:00:18,399 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:00:18,400 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:00:18,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:00:18,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:00:20,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:20,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:00:23,743 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:00:24,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:25,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:29,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:30,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:33,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:00:34,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:00:34,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:00:34,555 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:00:34,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:00:34,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:00:36,594 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:00:36,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:39,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:41,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:44,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:46,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:48,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:00:50,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:00:50,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:00:50,498 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:00:50,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:00:50,606 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:00:52,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:52,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:00:53,748 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:00:54,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:00:57,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:00:59,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:02,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:03,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:01:06,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:01:06,712 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:01:06,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:01:06,714 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:01:07,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:01:07,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:08,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:01:09,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:12,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:14,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:17,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:18,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:01:22,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:22,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:01:22,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:01:22,881 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:01:22,882 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:01:23,634 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:01:23,749 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:01:24,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:24,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:01:28,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:29,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:33,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:01:33,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:38,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:38,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:01:38,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:01:38,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:01:38,828 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:01:39,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:39,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:01:40,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:01:43,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:44,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:48,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:01:48,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:53,752 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:01:54,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:54,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:01:54,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:01:54,774 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:01:54,775 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:01:54,776 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:01:55,660 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:01:56,661 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:01:59,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:01:59,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:03,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:02:04,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:09,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:09,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:10,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:02:10,621 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:02:10,622 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:02:10,624 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:02:10,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:02:12,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:02:14,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:15,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:18,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:02:20,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:23,757 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:02:24,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:25,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:26,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:02:26,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:02:26,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:02:26,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:02:27,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:02:28,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:02:29,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:30,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:33,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:02:35,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:39,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:40,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:42,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:02:42,696 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:02:42,696 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:02:42,697 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:02:42,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:02:44,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:44,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:02:46,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:48,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:02:51,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:53,759 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:02:54,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:02:56,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:02:58,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:02:58,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:02:58,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:02:58,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:02:58,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:02:59,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:00,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:03:02,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:03,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:03:07,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:09,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:12,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:14,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:14,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:03:14,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:03:14,585 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:03:14,587 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:03:14,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:03:16,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:03:17,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:18,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:03:22,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:23,762 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:03:24,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:27,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:29,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:30,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:03:30,642 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:03:30,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:03:30,644 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:03:30,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:03:32,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:03:33,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:33,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:03:38,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:39,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:43,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:44,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:46,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:03:46,610 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:03:46,610 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:03:46,612 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:03:46,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:03:48,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:03:48,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:03:49,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:53,765 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:03:54,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:54,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:03:59,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:03:59,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:02,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:04:02,740 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:04:02,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:04:02,743 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:04:02,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:04:03,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:04:04,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:04:05,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:09,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:10,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:14,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:15,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:18,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:04:18,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:04:18,790 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:04:18,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:04:18,792 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:04:19,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:04:20,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:04:20,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:23,768 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:04:24,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:26,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:29,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:31,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:33,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:04:34,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:04:34,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:04:34,731 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:04:34,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:04:34,801 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:04:36,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:36,802 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:04:39,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:41,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:44,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:46,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:48,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:04:50,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:04:50,770 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:04:50,771 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:04:50,772 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:04:50,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:04:51,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:52,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:04:53,771 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:04:54,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:04:57,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:04:59,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:02,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:03,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:05:06,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:05:06,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:05:06,914 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:05:06,916 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:05:07,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:05:07,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:08,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:05:09,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:12,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:14,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:17,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:18,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:05:22,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:23,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:05:23,092 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:05:23,092 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:05:23,093 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:05:23,774 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:05:23,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:05:24,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:24,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:05:28,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:29,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:33,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:05:33,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:38,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:38,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:05:38,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:05:38,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:05:38,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:05:39,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:39,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:05:40,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:05:43,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:44,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:48,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:05:48,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:53,778 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:05:54,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:05:54,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:05:55,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:05:55,127 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:05:55,128 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:05:55,129 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:05:55,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:05:56,868 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:05:59,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:00,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:03,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:06:05,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:09,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:10,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:11,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:06:11,077 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:06:11,077 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:06:11,079 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:06:11,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:06:12,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:06:14,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:16,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:18,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:06:21,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:23,780 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:06:24,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:26,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:27,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:06:27,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:06:27,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:06:27,138 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:06:27,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:06:28,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:06:29,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:32,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:33,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:06:37,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:39,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:42,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:43,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:06:43,405 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:06:43,405 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:06:43,407 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:06:43,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:06:44,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:44,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:06:47,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:48,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:06:52,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:53,815 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:06:54,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:57,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:06:59,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:06:59,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:06:59,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:06:59,470 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:06:59,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:06:59,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:07:00,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:07:03,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:03,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:07:08,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:09,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:13,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:14,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:15,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:07:15,502 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:07:15,503 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:07:15,504 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:07:15,932 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:07:16,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:07:18,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:18,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:07:23,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:23,819 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:07:24,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:28,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:29,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:31,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:07:31,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:07:31,684 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:07:31,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:07:31,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:07:32,946 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:07:33,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:07:34,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:39,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:39,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:44,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:44,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:47,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:07:47,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:07:47,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:07:47,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:07:47,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:07:48,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:07:48,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:07:49,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:53,822 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:07:54,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:07:55,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:07:59,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:00,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:03,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:08:03,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:08:03,864 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:08:03,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:08:03,865 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:08:03,972 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:08:04,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:08:05,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:09,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:10,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:14,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:15,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:18,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:08:19,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:08:19,904 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:08:19,904 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:08:19,906 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:08:19,985 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:08:20,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:20,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:08:23,825 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:08:24,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:26,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:29,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:31,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:33,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:08:36,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:08:36,243 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:08:36,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:08:36,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:08:37,000 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:08:37,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:39,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:08:39,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:42,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:44,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:47,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:48,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:08:52,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:08:52,216 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:08:52,216 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:08:52,217 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:08:53,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:08:53,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:08:53,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:53,828 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:08:54,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:08:58,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:08:59,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:03,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:09:03,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:08,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:09:08,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:09:08,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:09:08,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:09:09,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:09:09,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:09,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:11,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:09:14,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:14,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:18,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:09:19,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:23,833 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:09:24,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:24,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:09:24,584 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:09:24,585 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:09:24,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:24,586 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:09:25,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:09:27,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:09:29,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:29,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:33,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:09:34,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:39,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:39,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:40,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:09:40,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:09:40,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:09:40,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:09:41,052 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:09:43,053 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:09:44,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:44,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:48,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:09:49,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:53,837 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:09:54,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:54,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:09:56,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:09:56,943 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:09:56,943 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:09:56,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:09:57,065 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:09:59,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:09:59,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:09:59,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:03,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:10:04,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:09,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:09,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:12,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:10:12,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:10:12,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:10:12,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:10:13,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:10:14,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:15,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:10:15,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:18,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:10:20,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:23,840 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:10:24,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:26,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:28,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:10:28,937 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:10:28,938 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:10:28,939 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:10:29,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:10:29,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:31,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:10:31,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:33,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:10:36,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:39,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:41,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:44,372 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:44,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:10:44,996 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:10:44,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:10:44,997 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:10:45,103 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:10:46,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:47,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:10:48,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:10:51,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:53,843 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:10:54,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:10:57,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:10:59,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:00,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:11:00,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:11:00,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:11:00,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:11:01,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:11:02,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:03,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:11:03,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:11:07,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:09,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:12,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:14,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:16,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:11:16,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:11:16,973 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:11:16,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:11:17,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:11:17,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:18,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:11:19,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:11:22,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:23,846 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:11:24,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:28,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:29,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:33,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:11:33,249 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:11:33,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:11:33,251 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:11:33,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:11:34,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:11:34,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:35,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:11:39,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:39,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:44,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:44,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:48,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:11:49,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:11:49,210 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:11:49,210 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:11:49,211 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:11:49,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:50,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:11:51,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:11:53,849 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:11:54,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:54,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:11:59,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:11:59,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:03,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:12:04,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:05,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:12:05,267 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:12:05,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:12:05,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:12:06,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:12:07,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:12:09,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:10,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:14,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:15,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:18,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:12:20,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:21,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:12:21,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:12:21,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:12:21,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:12:21,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:12:23,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:12:23,852 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:12:24,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:25,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:29,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:30,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:33,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:12:35,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:37,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:12:37,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:12:37,380 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:12:37,381 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:12:38,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:12:39,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:12:39,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:41,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:44,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:46,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:48,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:12:51,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:53,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:12:53,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:12:53,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:12:53,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:12:53,855 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:12:54,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:12:54,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:12:55,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:12:56,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:12:59,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:01,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:03,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:13:06,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:09,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:13:09,306 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:13:09,307 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:13:09,308 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:13:09,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:10,230 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:13:11,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:13:12,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:14,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:17,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:18,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:13:22,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:23,858 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:13:24,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:25,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:13:25,248 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:13:25,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:13:25,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:13:26,243 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:13:27,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:13:28,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:29,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:33,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:33,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:13:38,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:39,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:41,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:13:41,393 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:13:41,394 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:13:41,395 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:13:42,256 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:13:43,257 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:13:43,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:44,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:48,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:48,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:13:53,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:53,861 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:13:54,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:13:57,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:13:57,343 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:13:57,344 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:13:57,345 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:13:58,269 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:13:59,270 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:13:59,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:13:59,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:03,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:14:04,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:09,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:09,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:13,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:14:13,486 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:14:13,487 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:14:13,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:14:14,282 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:14:14,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:14,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:15,283 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:14:18,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:14:19,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:23,865 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:14:24,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:24,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:29,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:29,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:14:29,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:14:29,544 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:14:29,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:14:30,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:14:30,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:31,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:14:33,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:14:35,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:39,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:40,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:44,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:45,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:14:45,477 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:14:45,477 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:14:45,479 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:14:46,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:14:46,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:47,309 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:14:48,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:14:51,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:53,867 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:14:54,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:14:56,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:14:59,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:01,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:15:01,634 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:15:01,634 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:15:01,635 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:15:02,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:15:02,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:03,321 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:15:03,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:15:07,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:09,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:12,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:14,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:17,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:17,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:15:17,787 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:15:17,787 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:15:17,789 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:15:18,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:15:18,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:15:19,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:15:22,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:23,870 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:15:24,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:27,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:29,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:32,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:33,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:15:33,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:15:33,844 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:15:33,844 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:15:33,846 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:15:34,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:15:35,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:15:37,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:39,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:42,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:44,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:47,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:48,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:15:49,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:15:49,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:15:49,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:15:49,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:15:50,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:15:51,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:15:53,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:53,875 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:15:54,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:15:58,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:15:59,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:03,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:16:03,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:05,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:16:05,828 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:16:05,828 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:16:05,830 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:16:06,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:16:07,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:16:09,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:09,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:14,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:14,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:18,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:16:19,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:21,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:16:21,942 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:16:21,942 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:16:21,944 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:16:22,384 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:16:23,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:16:23,876 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:16:24,434 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:24,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:29,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:29,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:33,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:16:34,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:38,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:16:38,110 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:16:38,111 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:16:38,112 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:16:38,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:16:39,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:16:39,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:40,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:44,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:45,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:16:50,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:53,880 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:16:54,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:16:54,252 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:16:54,252 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:16:54,254 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:16:54,410 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:16:54,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:16:55,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:16:55,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:16:59,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:00,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:03,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:17:05,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:09,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:10,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:10,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:17:10,383 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:17:10,384 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:17:10,385 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:17:10,424 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:17:11,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:17:14,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:15,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:18,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:17:20,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:23,882 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:17:24,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:25,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:26,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:17:26,321 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:17:26,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:17:26,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:17:26,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:17:27,437 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:17:29,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:31,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:33,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:17:36,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:39,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:41,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:42,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:17:42,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:17:42,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:17:42,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:17:43,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:17:43,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:17:44,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:46,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:17:51,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:53,887 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:17:54,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:17:56,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:17:58,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:17:58,684 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:17:58,685 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:17:58,686 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:17:59,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:17:59,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:17:59,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:02,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:03,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:18:07,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:09,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:12,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:14,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:14,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:18:14,719 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:18:14,719 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:18:14,720 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:18:15,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:18:15,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:18:17,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:18,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:18:22,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:23,889 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:18:24,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:27,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:29,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:30,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:18:30,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:18:30,665 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:18:30,666 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:18:31,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:18:31,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:18:33,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:18:33,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:38,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:39,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:43,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:44,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:46,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:18:46,938 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:18:46,939 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:18:46,940 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:18:47,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:18:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:18:48,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:49,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:18:53,892 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:18:54,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:54,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:18:59,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:18:59,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:02,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:19:02,915 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:19:02,916 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:19:02,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:19:03,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:19:03,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:19:03,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:19:04,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:09,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:09,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:14,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:14,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:18,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:19:18,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:19:18,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:19:18,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:19:18,943 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:19:19,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:19:19,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:21,532 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:19:23,897 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:19:24,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:25,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:29,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:30,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:33,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:19:34,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:19:34,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:19:34,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:19:34,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:19:35,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:19:36,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:37,545 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:19:39,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:41,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:44,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:46,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:48,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:19:51,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:51,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:19:51,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:19:51,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:19:51,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:19:51,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:19:53,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:19:53,899 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:19:54,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:19:56,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:19:59,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:01,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:03,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:20:06,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:07,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:20:07,331 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:20:07,332 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:20:07,333 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:20:07,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:20:09,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:09,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:20:12,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:14,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:17,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:18,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:20:22,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:23,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:20:23,494 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:20:23,494 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:20:23,496 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:20:23,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:20:23,902 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:20:24,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:25,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:20:27,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:29,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:32,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:33,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:20:37,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:39,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:20:39,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:20:39,535 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:20:39,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:20:39,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:39,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:20:41,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:20:43,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:44,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:48,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:48,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:20:53,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:53,907 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:20:54,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:20:55,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:20:55,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:20:55,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:20:55,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:20:55,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:20:57,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:20:59,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:20:59,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:03,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:21:04,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:09,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:09,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:11,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:21:11,505 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:21:11,506 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:21:11,507 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:21:11,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:21:13,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:21:14,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:14,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:18,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:21:19,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:23,910 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:21:24,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:24,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:27,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:21:27,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:21:27,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:21:27,441 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:21:27,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:21:29,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:29,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:21:30,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:33,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:21:35,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:39,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:40,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:43,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:21:43,728 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:21:43,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:21:43,730 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:21:44,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:44,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:21:45,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:21:45,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:48,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:21:50,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:53,912 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:21:54,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:55,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:21:59,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:21:59,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:21:59,661 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:21:59,662 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:21:59,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:21:59,664 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:22:01,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:01,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:22:03,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:22:06,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:09,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:11,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:14,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:15,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:22:15,706 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:22:15,707 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:22:15,709 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:22:16,679 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:22:16,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:17,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:22:18,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:22:21,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:23,916 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:22:24,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:26,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:29,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:31,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:22:31,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:22:31,561 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:22:31,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:22:31,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:22:32,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:33,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:22:33,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:22:37,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:39,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:42,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:44,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:47,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:47,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:22:47,831 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:22:47,832 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:22:47,833 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:22:48,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:22:48,706 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:22:49,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:22:52,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:53,922 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:22:54,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:22:57,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:22:59,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:02,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:03,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:23:03,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:23:03,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:23:03,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:23:03,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:23:04,719 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:23:05,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:23:08,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:09,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:13,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:14,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:18,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:23:18,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:19,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:23:19,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:23:19,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:23:19,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:23:20,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:23:21,734 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:23:23,924 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:23:23,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:24,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:28,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:29,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:33,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:23:33,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:35,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:23:35,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:23:35,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:23:35,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:23:36,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:23:37,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:23:38,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:39,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:43,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:44,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:48,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:23:48,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:52,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:23:52,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:23:52,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:23:52,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:23:52,758 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:23:53,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:23:53,927 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:23:54,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:54,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:23:59,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:23:59,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:03,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:24:04,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:07,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:24:07,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:24:07,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:24:07,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:24:08,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:24:09,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:09,772 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:24:10,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:14,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:15,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:18,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:24:20,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:23,930 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:24:24,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:24:24,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:24:24,124 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:24:24,125 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:24:24,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:24,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:24:25,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:25,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:24:29,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:30,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:33,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:24:35,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:39,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:40,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:24:40,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:24:40,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:24:40,167 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:24:40,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:24:41,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:41,798 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:24:44,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:46,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:48,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:24:51,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:53,933 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:24:54,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:24:56,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:24:56,166 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:24:56,166 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:24:56,168 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:24:56,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:24:57,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:24:57,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:24:59,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:02,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:03,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:25:07,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:09,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:12,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:12,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:25:12,254 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:25:12,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:25:12,255 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:25:12,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:25:13,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:25:14,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:17,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:18,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:25:22,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:23,936 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:25:24,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:27,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:28,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:25:28,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:25:28,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:25:28,358 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:25:28,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:25:29,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:29,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:25:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:33,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:25:38,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:39,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:43,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:44,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:25:44,411 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:25:44,411 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:25:44,413 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:25:44,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:44,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:25:45,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:25:48,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:48,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:25:53,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:53,939 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:25:54,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:25:58,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:25:59,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:00,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:26:00,487 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:26:00,487 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:26:00,489 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:26:00,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:26:01,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:26:03,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:26:04,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:09,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:09,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:14,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:14,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:16,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:26:16,519 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:26:16,519 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:26:16,521 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:26:16,875 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:26:17,875 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:26:18,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:26:19,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:23,944 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:26:24,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:24,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:29,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:29,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:32,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:26:32,558 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:26:32,559 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:26:32,560 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:26:32,886 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:26:33,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:26:33,887 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:26:35,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:39,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:40,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:44,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:45,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:48,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:26:48,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:26:48,631 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:26:48,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:26:48,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:26:48,900 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:26:49,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:26:50,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:53,946 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:26:54,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:26:55,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:26:59,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:00,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:03,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:27:04,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:27:04,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:27:04,818 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:27:04,819 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:27:04,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:27:05,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:27:06,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:08,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:11,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:13,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:16,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:18,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:27:20,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:27:20,999 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:27:21,000 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:27:21,001 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:27:21,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:27:21,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:27:22,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:23,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:23,949 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:27:27,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:28,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:32,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:33,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:27:37,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:27:37,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:27:37,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:27:37,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:27:37,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:27:37,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:27:38,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:38,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:43,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:43,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:48,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:48,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:27:53,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:27:53,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:27:53,267 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:27:53,268 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:27:53,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:27:53,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:53,952 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:27:53,954 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:27:53,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:27:58,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:27:58,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:03,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:28:03,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:08,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:08,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:09,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:28:09,304 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:28:09,304 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:28:09,306 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:28:09,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:28:09,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:28:13,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:14,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:18,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:28:19,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:23,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:23,954 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:28:24,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:25,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:28:25,332 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:28:25,332 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:28:25,334 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:28:25,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:28:25,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:28:28,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:30,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:33,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:28:35,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:38,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:40,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:41,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:28:41,294 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:28:41,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:28:41,296 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:28:41,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:28:41,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:28:43,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:46,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:48,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:28:51,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:53,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:28:53,958 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:28:56,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:28:57,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:28:57,546 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:28:57,547 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:28:57,548 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:28:58,004 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:28:58,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:28:58,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:02,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:03,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:29:07,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:08,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:12,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:13,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:29:13,497 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:29:13,497 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:29:13,499 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:29:13,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:14,017 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:29:14,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:29:18,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:18,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:29:23,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:23,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:23,961 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:29:28,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:28,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:29,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:29:29,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:29:29,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:29:29,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:29:30,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:29:30,030 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:29:33,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:29:34,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:38,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:39,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:43,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:44,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:45,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:29:45,603 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:29:45,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:29:45,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:29:46,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:29:46,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:29:48,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:29:49,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:53,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:53,964 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:29:54,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:29:58,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:29:59,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:01,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:30:01,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:30:01,752 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:30:01,753 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:30:02,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:30:03,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:30:04,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:30:05,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:08,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:10,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:13,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:15,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:17,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:30:17,987 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:30:17,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:30:17,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:30:18,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:30:18,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:30:20,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:30:20,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:23,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:23,967 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:30:26,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:28,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:31,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:33,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:30:34,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:30:34,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:30:34,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:30:34,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:30:34,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:30:36,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:30:37,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:38,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:42,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:43,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:47,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:48,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:30:50,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:30:50,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:30:50,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:30:50,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:30:51,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:30:52,095 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:30:52,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:53,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:30:53,970 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:30:57,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:30:58,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:02,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:03,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:31:06,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:31:06,112 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:31:06,112 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:31:06,113 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:31:07,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:31:08,107 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:31:08,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:08,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:13,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:13,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:18,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:18,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:31:22,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:31:22,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:31:22,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:31:22,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:31:22,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:31:23,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:23,974 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:31:23,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:24,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:31:28,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:28,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:33,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:31:33,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:38,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:31:38,343 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:31:38,343 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:31:38,344 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:31:38,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:39,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:31:39,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:40,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:31:43,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:44,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:48,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:31:49,388 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:53,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:53,976 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:31:54,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:31:54,396 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:31:54,397 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:31:54,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:31:54,398 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:31:55,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:31:56,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:31:58,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:31:59,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:03,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:32:04,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:08,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:09,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:10,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:32:10,443 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:32:10,443 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:32:10,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:32:11,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:32:12,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:32:13,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:14,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:18,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:32:19,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:23,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:23,979 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:32:24,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:26,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:32:26,470 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:32:26,470 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:32:26,472 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:32:27,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:32:28,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:32:28,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:30,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:33,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:32:35,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:38,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:40,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:42,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:32:42,322 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:32:42,323 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:32:42,324 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:32:43,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:32:43,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:44,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:32:46,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:48,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:32:51,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:53,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:53,983 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:32:56,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:32:58,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:32:58,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:32:58,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:32:58,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:32:58,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:32:59,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:33:00,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:33:02,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:03,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:33:07,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:08,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:12,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:13,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:14,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:33:14,604 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:33:14,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:33:14,606 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:33:15,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:33:16,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:33:17,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:18,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:33:22,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:23,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:23,985 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:33:27,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:28,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:30,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:33:30,566 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:33:30,567 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:33:30,568 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:33:31,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:33:32,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:33:33,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:33,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:33:38,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:38,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:43,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:43,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:46,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:33:46,593 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:33:46,593 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:33:46,595 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:33:47,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:33:48,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:33:48,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:48,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:33:53,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:33:53,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:53,989 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:33:58,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:33:58,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:02,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:34:02,738 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:34:02,739 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:34:02,740 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:34:03,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:34:03,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:34:04,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:34:04,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:08,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:09,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:13,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:14,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:18,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:34:18,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:34:18,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:34:18,791 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:34:18,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:34:19,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:34:19,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:20,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:34:23,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:23,992 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:34:24,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:28,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:29,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:33,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:34:34,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:35,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:34:35,094 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:34:35,094 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:34:35,096 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:34:35,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:34:36,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:34:38,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:40,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:43,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:45,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:48,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:34:50,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:51,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:34:51,174 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:34:51,174 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:34:51,176 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:34:51,288 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:34:52,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:34:53,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:34:53,996 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:34:55,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:34:58,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:00,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:03,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:35:05,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:07,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:35:07,227 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:35:07,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:35:07,229 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:35:07,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:35:08,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:35:08,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:11,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:13,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:16,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:18,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:35:21,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:23,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:35:23,284 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:35:23,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:35:23,286 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:35:23,314 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:35:23,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:23,997 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:35:24,314 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:35:26,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:28,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:31,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:33,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:35:37,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:38,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:39,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:35:39,413 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:35:39,413 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:35:39,415 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:35:40,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:35:40,328 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:35:42,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:43,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:47,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:48,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:35:52,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:53,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:35:54,001 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:35:55,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:35:55,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:35:55,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:35:55,562 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:35:56,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:35:56,340 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:35:57,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:35:58,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:02,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:03,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:36:07,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:08,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:11,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:36:11,416 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:36:11,417 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:36:11,418 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:36:12,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:36:12,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:36:13,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:13,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:18,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:18,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:36:23,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:23,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:24,005 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:36:27,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:36:27,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:36:27,543 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:36:27,544 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:36:28,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:36:28,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:36:28,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:28,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:33,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:33,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:36:38,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:38,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:43,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:43,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:36:43,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:36:43,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:36:43,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:36:43,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:44,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:36:44,380 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:36:48,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:48,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:36:53,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:53,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:54,007 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:36:58,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:36:59,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:36:59,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:36:59,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:36:59,605 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:36:59,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:37:00,391 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:37:00,391 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:37:03,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:37:04,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:09,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:09,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:14,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:14,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:15,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:37:15,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:37:15,776 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:37:15,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:37:16,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:37:16,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:37:18,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:37:19,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:24,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:24,010 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:37:25,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:29,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:30,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:31,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:37:31,823 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:37:31,823 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:37:31,825 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:37:32,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:37:32,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:37:33,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:37:35,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:39,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:40,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:44,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:45,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:47,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:37:47,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:37:47,768 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:37:47,770 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:37:48,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:37:48,428 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:37:48,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:37:51,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:54,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:37:54,014 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:37:57,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:37:59,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:02,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:03,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:38:03,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:38:03,910 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:38:03,910 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:38:03,911 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:38:04,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:38:04,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:38:07,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:09,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:12,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:14,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:17,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:18,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:38:19,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:38:19,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:38:19,857 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:38:19,858 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:38:20,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:38:20,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:38:23,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:24,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:24,017 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:38:29,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:29,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:33,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:38:34,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:35,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:38:35,894 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:38:35,894 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:38:35,896 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:38:36,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:38:36,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:38:39,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:39,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:44,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:44,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:48,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:38:49,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:51,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:38:51,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:38:51,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:38:51,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:38:52,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:38:52,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:38:54,019 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:38:54,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:38:55,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:38:59,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:00,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:03,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:39:05,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:08,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:39:08,214 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:39:08,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:39:08,216 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:39:08,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:39:08,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:39:09,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:10,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:14,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:15,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:18,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:39:20,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:24,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:24,030 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:39:24,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:39:24,170 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:39:24,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:39:24,172 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:39:24,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:39:24,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:39:26,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:29,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:31,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:33,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:39:36,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:39,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:40,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:39:40,204 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:39:40,204 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:39:40,205 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:39:40,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:39:40,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:39:41,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:44,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:46,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:48,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:39:51,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:54,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:39:54,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:39:56,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:39:56,238 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:39:56,239 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:39:56,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:39:56,240 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:39:56,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:39:57,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:39:59,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:01,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:03,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:40:06,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:09,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:11,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:12,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:40:12,475 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:40:12,475 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:40:12,488 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:40:12,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:40:14,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:15,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:40:16,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:18,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:40:21,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:24,035 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:40:24,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:27,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:28,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:40:28,703 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:40:28,703 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:40:28,705 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:40:29,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:29,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:40:31,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:40:32,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:33,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:40:37,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:39,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:42,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:44,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:44,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:40:44,640 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:40:44,640 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:40:44,642 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:40:45,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:40:47,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:40:48,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:48,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:40:53,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:54,039 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:40:54,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:40:59,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:40:59,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:00,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:41:00,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:41:00,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:41:00,785 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:41:01,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:41:03,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:41:03,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:41:04,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:09,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:09,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:14,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:14,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:16,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:41:16,724 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:41:16,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:41:16,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:41:17,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:41:18,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:41:19,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:41:20,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:24,042 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:41:24,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:26,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:29,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:31,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:32,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:41:32,674 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:41:32,674 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:41:32,676 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:41:33,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:41:33,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:41:35,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:41:36,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:39,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:41,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:44,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:46,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:48,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:41:48,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:41:48,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:41:48,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:41:48,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:41:49,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:41:51,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:41:51,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:54,079 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:41:54,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:41:57,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:41:59,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:02,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:03,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:42:04,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:42:04,895 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:42:04,895 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:42:04,896 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:42:05,634 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:42:07,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:42:07,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:09,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:12,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:14,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:17,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:18,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:42:20,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:42:20,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:42:20,836 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:42:20,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:42:21,647 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:42:23,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:42:23,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:24,082 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:42:24,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:29,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:29,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:33,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:42:34,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:36,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:42:36,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:42:36,998 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:42:36,999 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:42:37,661 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:42:39,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:39,662 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:42:40,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:44,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:45,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:48,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:42:50,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:52,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:42:52,861 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:42:52,861 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:42:52,862 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:42:53,674 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:42:54,085 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:42:54,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:42:55,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:42:55,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:42:57,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:42:59,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:00,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:01,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:03,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:03,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:43:04,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:05,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:09,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:09,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:11,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:11,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:13,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:43:14,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:15,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:17,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:18,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:43:22,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:23,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:24,089 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:43:28,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:28,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:29,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:43:29,527 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:43:29,528 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:43:29,529 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:43:29,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:43:31,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:33,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:33,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:43:38,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:38,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:43,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:43,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:45,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:43:45,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:43:45,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:43:45,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:43:46,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:43:47,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:43:48,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:43:48,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:53,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:53,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:43:54,092 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:43:58,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:43:59,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:01,803 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:44:01,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:44:01,806 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:44:01,808 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:44:02,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:44:03,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:44:03,731 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:44:04,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:08,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:09,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:13,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:14,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:18,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:44:18,041 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:44:18,042 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:44:18,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:44:18,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:44:18,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:44:19,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:44:20,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:23,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:24,095 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:44:25,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:28,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:30,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:33,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:44:34,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:44:34,201 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:44:34,202 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:44:34,203 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:44:34,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:44:35,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:35,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:44:38,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:40,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:43,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:45,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:48,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:44:50,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:44:50,140 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:44:50,140 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:44:50,142 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:44:50,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:44:51,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:51,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:44:53,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:44:54,098 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:44:57,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:44:58,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:02,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:03,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:45:06,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:45:06,069 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:45:06,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:45:06,071 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:45:06,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:45:07,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:07,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:45:08,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:12,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:13,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:17,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:18,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:45:22,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:22,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:45:22,120 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:45:22,120 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:45:22,122 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:45:22,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:45:23,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:45:23,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:24,100 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:45:28,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:28,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:33,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:33,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:45:37,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:45:37,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:45:37,965 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:45:37,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:45:38,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:45:38,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:39,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:39,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:45:43,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:44,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:48,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:45:49,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:53,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:54,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:45:54,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:45:54,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:45:54,017 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:45:54,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:45:54,103 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:45:54,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:45:55,822 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:45:58,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:45:59,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:03,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:46:04,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:08,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:09,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:09,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:46:09,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:46:09,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:46:09,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:46:10,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:46:11,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:46:13,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:14,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:18,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:46:19,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:23,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:24,107 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:46:25,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:25,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:46:25,907 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:46:25,908 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:46:25,909 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:46:26,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:46:27,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:46:28,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:30,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:33,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:46:35,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:38,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:40,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:41,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:46:41,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:46:41,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:46:41,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:46:42,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:46:43,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:46:43,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:45,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:48,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:46:50,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:53,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:54,110 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:46:56,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:46:58,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:46:58,094 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:46:58,095 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:46:58,096 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:46:58,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:46:58,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:46:59,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:47:02,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:03,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:47:07,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:08,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:12,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:13,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:14,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:47:14,153 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:47:14,154 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:47:14,155 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:47:14,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:47:15,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:47:17,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:18,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:47:22,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:23,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:24,113 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:47:28,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:28,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:30,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:47:30,112 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:47:30,112 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:47:30,114 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:47:30,903 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:47:31,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:47:33,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:47:34,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:38,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:39,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:43,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:44,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:46,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:47:46,251 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:47:46,251 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:47:46,253 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:47:46,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:47:47,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:47:48,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:47:49,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:53,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:47:54,116 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:47:55,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:47:58,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:00,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:02,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:48:02,222 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:48:02,223 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:48:02,224 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:48:02,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:48:03,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:48:03,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:48:05,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:08,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:10,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:13,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:15,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:18,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:48:18,585 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:48:18,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:48:18,587 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:48:18,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:48:18,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:48:19,943 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:48:20,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:23,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:24,120 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:48:26,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:28,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:31,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:33,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:48:34,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:48:34,445 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:48:34,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:48:34,447 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:48:34,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:48:35,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:48:36,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:38,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:41,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:43,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:46,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:48,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:48:50,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:48:50,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:48:50,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:48:50,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:48:50,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:48:51,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:51,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:48:53,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:48:54,122 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:48:57,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:48:58,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:02,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:03,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:49:06,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:49:06,847 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:49:06,847 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:49:06,849 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:49:06,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:49:07,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:07,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:49:08,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:12,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:13,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:17,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:18,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:49:22,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:49:22,793 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:49:22,794 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:49:22,795 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:49:22,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:49:23,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:23,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:23,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:49:24,125 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:49:28,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:29,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:33,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:49:34,128 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:38,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:39,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:49:39,080 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:49:39,080 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:49:39,082 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:49:40,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:49:40,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:42,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:49:43,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:45,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:48,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:49:50,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:53,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:49:54,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:49:55,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:49:55,021 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:49:55,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:49:55,023 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:49:56,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:49:56,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:49:58,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:49:58,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:01,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:03,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:50:06,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:09,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:11,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:11,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:50:11,149 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:50:11,149 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:50:11,151 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:50:12,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:50:14,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:14,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:50:16,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:18,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:50:21,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:24,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:24,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:50:27,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:27,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:50:27,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:50:27,566 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:50:27,568 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:50:28,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:50:29,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:30,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:50:32,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:33,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:50:37,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:39,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:42,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:43,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:50:43,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:50:43,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:50:43,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:50:44,059 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:50:44,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:46,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:50:48,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:48,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:50:53,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:54,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:54,134 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:50:59,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:50:59,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:50:59,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:50:59,491 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:50:59,492 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:50:59,494 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:51:00,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:51:02,073 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:51:03,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:51:04,495 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:09,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:09,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:14,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:14,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:15,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:51:15,738 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:51:15,738 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:51:15,740 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:51:16,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:51:18,095 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:51:18,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:51:19,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:24,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:24,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:51:25,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:29,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:30,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:31,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:51:31,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:51:31,834 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:51:31,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:51:32,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:51:33,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:51:34,108 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:51:35,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:39,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:40,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:44,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:45,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:47,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:51:47,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:51:47,777 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:51:47,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:51:48,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:51:48,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:51:50,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:51:51,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:54,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:51:54,244 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:51:57,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:51:59,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:02,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:03,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:52:04,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:52:04,124 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:52:04,125 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:52:04,127 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:52:04,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:52:06,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:52:08,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:09,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:13,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:14,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:18,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:18,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:52:20,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:52:20,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:52:20,167 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:52:20,168 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:52:21,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:52:22,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:52:23,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:24,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:24,143 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:52:29,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:29,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:33,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:52:34,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:36,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:52:36,440 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:52:36,440 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:52:36,442 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:52:37,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:52:38,158 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:52:39,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:39,486 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:44,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:44,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:48,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:52:49,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:52,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:52:52,607 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:52:52,607 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:52:52,609 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:52:53,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:52:54,146 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:52:54,171 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:52:54,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:52:55,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:52:59,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:00,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:03,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:53:05,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:09,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:53:09,006 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:53:09,007 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:53:09,009 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:53:09,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:53:09,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:10,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:53:11,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:14,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:16,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:18,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:53:21,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:24,151 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:53:24,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:24,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:53:24,946 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:53:24,947 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:53:24,948 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:53:25,199 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:53:26,200 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:53:26,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:29,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:31,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:33,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:53:36,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:39,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:41,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:53:41,168 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:53:41,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:53:41,170 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:53:41,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:53:42,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:42,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:53:44,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:47,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:48,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:53:52,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:54,153 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:53:54,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:53:57,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:53:57,577 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:53:57,577 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:53:57,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:53:57,579 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:53:58,227 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:53:59,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:00,229 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:54:02,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:03,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:54:07,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:09,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:12,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:13,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:54:13,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:54:13,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:54:13,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:54:14,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:14,253 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:54:16,255 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:54:18,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:18,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:54:23,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:24,156 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:54:24,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:29,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:29,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:29,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:54:29,757 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:54:29,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:54:29,759 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:54:30,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:54:32,266 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:54:33,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:54:34,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:39,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:39,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:44,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:44,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:45,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:54:45,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:54:45,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:54:45,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:54:46,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:54:48,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:54:48,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:54:49,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:54,161 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:54:54,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:54:55,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:54:59,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:00,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:01,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:55:01,824 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:55:01,824 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:55:01,826 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:55:02,287 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:55:03,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:55:04,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:55:05,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:09,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:10,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:14,249 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:15,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:17,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:55:17,748 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:55:17,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:55:17,751 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:55:18,299 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:55:18,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:55:20,301 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:55:21,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:24,165 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:55:24,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:27,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:29,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:32,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:33,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:55:33,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:55:33,928 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:55:33,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:55:33,930 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:55:34,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:55:36,314 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:55:37,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:39,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:42,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:44,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:47,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:48,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:55:49,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:55:49,777 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:55:49,778 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:55:49,780 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:55:50,327 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:55:52,328 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:55:53,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:54,168 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:55:54,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:55:59,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:55:59,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:03,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:56:04,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:05,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:56:05,731 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:56:05,731 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:56:05,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:56:06,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:56:08,342 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:56:09,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:09,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:14,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:14,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:18,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:56:19,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:21,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:56:21,794 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:56:21,794 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:56:21,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:56:22,355 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:56:24,170 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:56:24,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:24,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:56:25,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:29,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:30,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:33,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:56:35,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:37,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:56:37,766 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:56:37,767 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:56:37,768 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:56:38,369 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:56:39,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:40,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:56:40,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:44,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:45,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:48,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:56:50,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:53,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:56:53,713 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:56:53,714 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:56:53,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:56:54,174 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:56:54,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:56:54,383 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:56:56,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:56:56,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:56:59,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:01,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:03,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:57:06,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:09,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:09,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:57:09,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:57:09,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:57:09,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:57:10,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:57:11,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:12,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:57:14,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:16,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:18,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:57:21,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:24,178 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:57:24,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:25,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:57:25,929 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:57:25,929 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:57:25,931 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:57:26,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:57:27,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:28,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:57:29,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:32,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:33,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:57:37,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:39,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:41,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:57:41,877 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:57:41,878 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:57:41,880 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:57:42,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:57:43,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:44,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:44,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:57:48,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:57:48,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:53,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:54,182 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:57:54,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:57:58,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:57:58,028 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:57:58,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:57:58,030 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:57:58,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:57:59,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:57:59,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:00,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:58:03,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:58:04,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:09,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:09,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:13,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:58:13,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:58:13,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:58:13,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:58:14,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:58:14,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:14,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:16,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:58:18,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:58:19,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:23,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:24,185 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:58:25,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:28,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:30,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:30,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:58:30,777 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:58:30,777 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:58:30,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:58:31,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:58:32,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:58:33,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:58:35,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:38,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:40,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:43,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:45,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:46,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:58:46,618 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:58:46,619 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:58:46,620 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:58:47,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:58:48,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:58:48,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:58:51,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:53,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:58:54,228 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:58:57,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:58:58,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:02,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:02,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:59:02,760 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:59:02,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:59:02,762 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:59:03,493 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:59:03,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:59:04,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:59:07,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:08,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:12,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:13,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:17,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:18,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:59:19,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:59:19,022 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:59:19,022 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:59:19,024 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:59:19,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:59:20,508 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:59:23,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:23,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:24,231 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:59:28,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:28,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:33,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:33,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:59:34,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:59:34,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:59:34,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:59:34,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:59:35,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:59:36,521 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:59:38,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:39,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:43,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:44,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:48,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 13:59:49,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:51,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 13:59:51,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 13:59:51,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 13:59:51,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 13:59:51,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 13:59:52,535 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 13:59:53,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:54,234 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 13:59:54,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 13:59:58,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 13:59:59,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:03,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:00:04,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:07,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:00:07,158 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:00:07,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:00:07,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:00:07,549 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:00:08,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:00:08,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:10,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:13,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:15,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:18,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:00:20,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:23,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:00:23,382 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:00:23,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:00:23,384 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:00:23,563 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:00:23,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:24,237 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:00:24,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:00:25,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:28,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:30,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:33,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:00:35,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:38,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:39,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:00:39,724 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:00:39,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:00:39,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:00:40,578 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:00:40,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:42,580 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:00:43,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:45,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:48,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:00:50,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:53,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:00:54,240 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:00:55,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:00:55,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:00:55,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:00:55,695 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:00:56,592 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:00:56,593 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:00:56,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:00:58,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:01,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:03,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:01:06,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:08,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:11,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:01:11,646 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:01:11,647 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:01:11,649 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:01:12,607 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:01:12,608 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:01:12,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:13,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:17,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:18,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:01:22,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:23,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:24,244 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:01:27,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:01:27,885 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:01:27,885 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:01:27,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:27,887 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:01:28,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:01:28,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:30,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:01:32,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:33,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:01:37,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:38,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:42,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:43,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:43,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:01:43,941 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:01:43,941 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:01:43,943 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:01:44,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:01:46,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:01:47,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:48,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:01:52,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:53,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:54,247 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:01:58,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:01:58,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:01:59,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:01:59,882 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:01:59,883 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:01:59,884 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:02:00,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:02:02,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:02:03,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:02:03,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:08,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:08,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:13,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:13,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:16,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:02:16,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:02:16,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:02:16,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:02:16,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:02:18,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:02:18,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:02:19,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:23,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:24,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:24,250 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:02:28,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:29,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:32,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:02:32,281 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:02:32,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:02:32,283 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:02:32,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:02:33,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:02:34,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:34,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:02:38,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:39,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:43,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:44,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:48,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:02:48,511 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:02:48,511 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:02:48,513 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:02:48,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:02:48,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:02:49,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:50,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:02:53,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:02:54,253 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:02:55,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:02:58,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:00,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:03,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:03:04,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:03:04,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:03:04,664 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:03:04,666 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:03:04,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:03:05,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:06,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:03:08,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:10,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:13,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:15,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:18,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:03:20,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:20,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:03:21,000 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:03:21,000 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:03:21,002 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:03:21,721 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:03:22,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:03:23,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:24,256 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:03:26,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:28,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:31,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:33,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:03:36,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:37,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:03:37,036 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:03:37,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:03:37,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:03:37,734 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:03:38,735 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:03:38,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:42,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:43,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:47,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:48,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:03:52,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:53,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:03:53,269 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:03:53,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:03:53,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:03:53,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:03:53,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:03:54,262 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:03:54,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:03:57,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:03:58,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:02,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:03,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:04:07,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:08,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:09,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:04:09,785 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:04:09,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:04:09,787 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:04:10,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:04:10,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:04:12,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:13,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:17,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:18,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:04:22,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:23,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:24,262 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:04:25,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:04:25,735 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:04:25,736 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:04:25,737 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:04:25,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:04:26,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:04:28,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:28,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:33,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:04:33,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:38,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:38,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:41,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:04:41,947 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:04:41,948 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:04:41,949 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:04:42,790 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:04:42,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:04:43,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:43,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:48,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:04:48,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:53,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:53,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:04:54,265 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:04:58,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:04:58,045 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:04:58,045 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:04:58,046 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:04:58,804 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:04:58,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:04:59,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:00,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:05:03,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:05:04,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:08,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:09,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:13,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:13,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:05:13,988 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:05:13,988 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:05:13,990 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:05:14,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:05:14,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:05:14,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:18,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:05:19,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:23,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:24,269 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:05:25,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:28,891 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:29,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:05:29,908 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:05:29,909 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:05:29,910 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:05:30,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:05:30,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:05:30,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:33,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:05:35,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:38,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:40,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:43,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:45,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:45,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:05:45,942 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:05:45,943 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:05:45,944 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:05:46,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:05:46,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:05:48,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:05:50,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:53,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:05:54,273 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:05:56,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:05:58,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:01,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:01,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:06:01,791 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:06:01,792 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:06:01,793 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:06:01,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:06:02,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:06:03,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:06:06,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:08,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:11,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:13,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:16,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:17,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:06:17,728 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:06:17,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:06:17,730 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:06:17,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:06:18,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:06:18,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:06:22,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:23,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:24,275 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:06:28,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:28,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:33,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:33,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:06:33,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:06:33,889 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:06:33,889 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:06:33,892 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:06:33,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:06:34,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:06:38,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:38,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:43,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:43,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:48,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:06:48,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:49,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:06:49,846 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:06:49,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:06:49,848 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:06:49,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:06:50,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:06:53,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:54,279 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:06:54,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:06:58,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:06:59,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:03,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:07:04,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:05,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:07:05,793 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:07:05,794 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:07:05,795 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:07:05,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:07:06,921 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:07:08,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:09,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:13,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:14,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:18,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:07:19,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:21,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:07:21,943 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:07:21,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:07:21,945 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:07:22,934 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:07:22,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:07:23,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:24,280 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:07:25,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:28,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:30,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:33,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:07:35,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:37,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:07:37,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:07:37,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:07:37,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:07:37,948 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:07:38,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:38,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:07:40,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:43,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:45,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:48,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:07:50,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:53,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:07:54,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:07:54,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:07:54,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:07:54,077 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:07:54,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:07:54,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:07:54,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:07:56,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:07:58,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:01,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:03,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:08:06,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:08,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:10,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:08:10,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:08:10,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:08:10,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:08:10,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:08:12,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:12,979 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:08:13,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:17,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:18,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:08:22,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:23,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:24,286 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:08:26,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:08:26,127 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:08:26,128 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:08:26,129 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:08:26,991 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:08:26,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:08:28,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:28,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:33,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:33,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:08:38,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:38,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:42,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:08:42,454 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:08:42,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:08:42,456 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:08:43,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:08:43,457 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:43,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:45,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:08:48,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:48,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:08:53,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:08:53,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:54,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:08:58,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:08:58,407 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:08:58,407 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:08:58,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:08:58,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:08:59,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:08:59,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:01,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:09:03,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:09:04,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:08,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:09,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:13,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:14,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:14,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:09:14,560 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:09:14,560 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:09:14,561 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:09:15,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:09:17,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:09:18,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:09:19,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:23,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:24,293 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:09:25,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:28,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:30,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:30,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:09:30,722 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:09:30,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:09:30,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:09:31,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:09:33,050 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:09:33,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:09:35,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:38,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:40,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:43,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:45,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:46,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:09:46,776 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:09:46,776 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:09:46,778 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:09:47,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:09:48,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:09:49,063 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:09:50,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:53,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:09:54,296 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:09:56,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:09:58,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:01,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:03,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:10:03,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:10:03,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:10:03,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:10:03,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:10:03,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:10:05,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:10:07,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:08,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:12,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:13,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:17,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:18,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:10:19,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:10:19,150 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:10:19,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:10:19,152 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:10:20,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:10:21,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:10:22,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:23,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:24,300 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:10:27,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:28,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:32,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:33,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:10:35,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:10:35,184 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:10:35,184 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:10:35,186 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:10:36,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:10:37,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:10:38,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:38,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:43,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:43,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:48,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:48,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:10:51,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:10:51,631 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:10:51,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:10:51,633 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:10:52,118 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:10:53,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:10:53,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:10:53,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:54,302 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:10:58,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:10:59,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:03,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:11:04,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:07,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:11:07,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:11:07,582 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:11:07,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:11:08,132 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:11:08,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:09,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:11:09,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:13,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:14,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:18,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:11:19,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:23,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:11:23,548 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:11:23,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:11:23,550 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:11:23,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:24,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:11:24,305 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:11:25,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:11:25,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:28,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:30,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:33,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:11:35,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:38,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:39,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:11:39,982 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:11:39,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:11:39,984 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:11:40,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:11:40,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:41,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:11:43,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:45,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:48,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:11:50,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:53,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:11:54,308 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:11:55,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:11:55,951 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:11:55,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:11:55,953 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:11:56,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:11:56,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:11:57,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:11:58,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:01,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:03,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:12:06,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:08,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:11,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:12,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:12:12,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:12:12,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:12:12,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:12:12,188 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:12:13,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:12:13,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:17,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:18,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:12:22,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:23,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:24,311 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:12:27,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:28,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:12:28,281 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:12:28,282 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:12:28,283 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:12:28,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:29,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:12:29,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:12:33,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:33,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:12:38,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:38,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:43,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:43,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:44,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:12:44,520 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:12:44,520 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:12:44,522 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:12:45,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:12:47,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:12:48,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:48,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:12:53,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:12:53,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:54,313 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:12:58,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:12:59,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:00,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:13:00,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:13:00,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:13:00,555 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:13:01,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:13:03,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:13:03,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:13:04,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:08,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:09,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:13,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:14,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:16,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:13:16,909 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:13:16,910 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:13:16,911 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:13:17,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:13:18,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:13:19,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:13:19,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:23,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:24,317 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:13:25,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:28,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:30,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:33,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:13:33,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:13:33,169 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:13:33,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:13:33,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:13:33,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:13:35,264 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:13:36,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:38,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:41,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:43,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:46,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:48,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:13:49,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:13:49,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:13:49,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:13:49,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:13:49,277 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:13:51,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:13:52,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:53,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:13:54,321 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:13:57,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:13:58,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:02,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:03,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:14:05,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:14:05,249 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:14:05,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:14:05,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:14:05,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:14:07,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:14:08,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:08,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:13,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:13,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:18,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:18,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:14:21,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:14:21,554 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:14:21,554 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:14:21,556 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:14:22,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:14:23,306 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:14:23,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:23,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:24,323 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:14:28,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:29,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:33,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:14:34,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:37,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:14:37,512 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:14:37,512 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:14:37,514 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:14:38,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:14:38,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:39,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:14:39,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:43,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:44,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:48,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:14:49,559 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:53,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:14:53,639 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:14:53,639 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:14:53,640 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:14:53,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:14:54,326 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:14:54,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:14:55,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:14:55,334 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:14:58,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:00,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:03,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:15:05,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:08,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:09,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:15:09,781 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:15:09,781 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:15:09,783 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:15:10,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:15:10,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:11,348 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:15:13,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:15,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:18,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:15:20,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:23,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:24,329 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:15:25,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:15:25,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:15:25,918 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:15:25,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:25,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:15:26,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:15:27,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:15:28,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:30,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:33,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:15:35,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:38,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:40,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:41,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:15:41,853 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:15:41,854 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:15:41,855 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:15:42,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:15:43,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:15:43,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:46,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:48,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:15:51,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:53,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:54,333 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:15:57,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:15:57,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:15:57,997 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:15:57,998 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:15:58,000 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:15:58,388 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:15:58,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:15:59,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:16:03,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:03,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:16:08,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:08,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:13,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:13,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:16:13,845 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:16:13,845 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:16:13,847 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:16:13,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:14,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:16:15,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:16:18,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:16:18,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:23,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:23,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:24,336 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:16:28,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:29,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:29,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:16:29,893 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:16:29,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:16:29,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:16:30,415 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:16:31,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:16:33,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:16:34,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:38,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:39,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:43,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:44,897 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:45,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:16:45,940 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:16:45,940 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:16:45,942 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:16:46,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:16:47,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:16:48,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:16:49,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:53,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:16:54,372 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:16:55,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:16:58,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:00,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:01,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:17:01,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:17:01,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:17:01,805 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:17:02,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:17:03,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:17:03,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:17:05,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:08,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:10,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:13,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:15,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:17,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:17:17,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:17:17,851 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:17:17,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:17:18,456 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:17:18,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:17:19,457 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:17:20,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:23,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:24,377 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:17:26,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:28,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:31,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:33,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:17:34,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:17:34,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:17:34,135 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:17:34,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:17:34,470 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:17:35,471 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:17:37,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:39,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:42,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:44,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:47,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:48,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:17:50,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:17:50,073 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:17:50,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:17:50,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:17:50,486 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:17:51,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:17:53,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:54,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:17:54,379 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:17:58,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:17:59,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:03,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:03,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:18:06,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:18:06,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:18:06,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:18:06,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:18:06,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:18:07,500 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:18:09,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:09,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:14,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:14,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:18,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:18:19,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:22,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:18:22,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:18:22,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:18:22,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:18:22,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:18:23,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:18:24,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:24,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:24,381 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:18:29,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:29,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:33,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:18:34,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:38,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:18:38,117 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:18:38,117 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:18:38,119 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:18:38,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:18:39,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:39,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:18:40,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:44,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:45,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:48,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:18:50,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:54,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:18:54,385 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:18:54,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:18:54,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:18:54,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:18:54,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:18:55,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:18:55,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:18:55,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:18:59,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:00,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:03,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:19:05,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:09,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:10,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:19:10,416 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:19:10,416 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:19:10,418 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:19:10,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:19:11,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:11,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:19:14,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:16,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:18,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:19:21,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:24,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:24,387 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:19:26,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:19:26,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:19:26,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:19:26,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:26,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:19:27,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:19:27,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:19:29,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:31,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:33,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:19:36,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:39,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:41,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:42,718 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:19:42,721 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:19:42,721 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:19:42,723 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:19:43,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:19:43,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:19:44,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:46,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:48,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:19:51,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:54,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:54,391 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:19:57,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:19:58,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:19:58,882 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:19:58,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:19:58,884 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:19:59,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:19:59,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:19:59,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:20:02,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:03,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:20:07,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:09,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:12,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:14,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:15,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:20:15,165 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:20:15,165 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:20:15,166 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:20:15,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:20:17,614 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:20:18,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:18,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:20:23,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:24,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:24,395 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:20:28,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:29,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:31,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:20:31,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:20:31,114 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:20:31,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:20:31,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:20:33,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:20:33,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:20:34,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:39,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:39,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:44,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:44,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:47,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:20:47,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:20:47,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:20:47,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:20:47,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:20:48,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:20:49,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:49,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:20:54,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:54,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:20:54,397 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:20:59,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:20:59,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:03,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:21:03,667 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:21:03,668 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:21:03,669 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:21:03,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:21:04,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:21:04,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:05,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:21:09,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:09,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:14,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:14,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:18,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:21:19,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:21:19,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:21:19,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:21:19,618 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:21:19,668 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:21:20,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:21,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:21:24,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:24,401 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:21:26,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:29,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:31,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:33,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:21:35,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:21:35,664 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:21:35,664 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:21:35,665 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:21:35,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:21:36,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:37,696 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:21:39,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:41,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:44,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:46,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:48,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:21:51,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:51,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:21:51,915 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:21:51,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:21:51,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:21:52,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:21:53,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:21:54,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:21:54,404 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:21:57,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:21:59,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:02,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:03,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:22:07,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:07,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:22:07,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:22:07,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:22:07,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:22:08,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:22:09,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:09,720 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:22:12,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:14,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:17,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:18,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:22:22,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:23,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:22:23,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:22:23,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:22:23,823 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:22:24,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:24,408 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:22:24,732 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:22:25,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:22:28,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:29,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:33,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:33,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:22:38,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:39,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:39,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:22:39,972 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:22:39,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:22:39,974 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:22:40,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:22:41,744 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:22:43,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:44,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:48,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:22:48,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:53,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:54,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:22:54,410 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:22:56,134 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:22:56,137 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:22:56,137 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:22:56,139 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:22:56,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:22:57,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:22:59,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:22:59,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:03,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:23:04,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:09,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:09,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:12,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:23:12,177 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:23:12,178 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:23:12,179 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:23:12,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:23:13,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:23:14,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:14,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:18,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:23:19,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:24,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:24,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:24,413 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:23:28,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:23:28,448 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:23:28,448 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:23:28,449 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:23:28,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:23:29,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:29,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:29,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:23:33,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:23:34,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:39,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:39,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:44,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:44,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:44,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:23:44,694 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:23:44,694 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:23:44,695 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:23:44,793 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:23:45,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:23:48,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:23:49,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:54,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:23:54,417 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:23:55,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:23:59,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:00,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:00,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:24:00,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:24:00,733 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:24:00,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:24:00,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:24:01,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:24:03,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:24:05,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:09,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:10,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:14,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:15,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:16,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:24:16,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:24:16,981 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:24:16,982 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:24:17,818 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:24:17,819 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:24:18,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:24:20,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:24,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:24,419 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:24:26,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:29,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:31,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:33,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:24:33,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:24:33,293 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:24:33,294 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:24:33,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:24:33,832 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:24:35,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:24:37,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:39,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:42,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:44,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:47,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:48,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:24:49,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:24:49,327 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:24:49,327 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:24:49,329 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:24:49,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:24:51,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:24:52,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:54,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:24:54,423 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:24:57,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:24:59,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:02,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:03,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:25:05,445 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:25:05,447 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:25:05,447 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:25:05,449 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:25:05,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:25:07,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:07,863 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:25:09,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:12,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:14,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:17,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:18,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:25:21,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:25:21,562 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:25:21,562 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:25:21,563 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:25:21,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:25:22,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:23,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:25:24,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:24,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:25:28,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:29,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:33,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:33,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:25:37,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:25:37,605 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:25:37,605 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:25:37,607 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:25:37,890 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:25:38,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:39,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:39,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:25:43,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:44,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:48,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:48,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:25:53,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:25:53,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:25:53,543 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:25:53,545 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:25:53,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:25:54,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:54,431 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:25:54,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:25:55,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:25:59,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:25:59,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:03,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:26:04,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:09,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:09,433 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:09,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:26:09,580 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:26:09,581 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:26:09,582 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:26:09,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:26:11,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:26:14,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:14,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:18,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:26:19,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:24,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:24,432 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:26:25,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:26:25,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:26:25,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:26:25,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:25,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:26:25,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:26:27,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:26:29,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:30,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:33,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:26:35,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:39,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:40,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:41,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:26:41,521 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:26:41,522 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:26:41,523 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:26:41,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:26:43,947 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:26:44,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:45,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:48,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:26:50,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:54,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:54,435 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:26:56,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:26:57,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:26:57,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:26:57,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:26:57,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:26:57,959 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:26:59,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:26:59,961 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:27:01,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:03,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:27:06,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:09,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:11,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:13,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:27:13,519 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:27:13,519 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:27:13,521 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:27:13,973 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:27:14,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:15,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:27:17,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:18,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:27:22,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:24,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:24,440 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:27:28,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:29,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:29,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:27:29,563 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:27:29,563 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:27:29,565 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:27:29,986 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:27:31,988 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:27:33,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:33,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:27:38,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:39,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:43,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:44,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:45,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:27:45,714 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:27:45,714 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:27:45,716 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:27:46,001 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:27:48,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:27:48,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:27:48,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:53,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:27:54,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:54,441 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:27:59,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:27:59,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:01,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:28:01,661 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:28:01,662 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:28:01,663 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:28:02,014 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:28:03,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:28:04,016 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:28:04,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:09,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:09,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:14,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:14,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:17,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:28:17,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:28:17,825 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:28:17,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:28:18,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:28:18,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:28:19,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:20,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:28:24,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:24,445 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:28:25,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:29,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:30,446 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:33,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:28:33,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:28:33,980 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:28:33,980 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:28:33,982 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:28:34,041 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:28:36,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:36,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:28:39,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:41,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:44,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:46,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:48,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:28:49,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:28:49,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:28:49,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:28:49,851 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:28:50,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:28:51,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:52,058 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:28:54,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:28:54,448 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:28:57,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:28:59,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:02,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:03,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:29:06,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:29:06,192 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:29:06,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:29:06,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:29:07,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:29:08,071 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:29:08,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:09,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:13,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:14,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:18,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:18,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:29:22,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:29:22,162 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:29:22,162 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:29:22,164 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:29:23,083 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:29:24,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:29:24,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:24,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:24,451 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:29:29,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:29,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:33,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:29:34,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:38,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:29:38,423 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:29:38,424 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:29:38,425 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:29:39,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:29:39,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:39,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:40,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:29:44,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:44,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:48,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:29:49,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:54,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:29:54,454 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:29:54,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:29:54,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:29:54,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:29:54,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:29:55,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:29:55,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:29:56,111 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:29:59,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:00,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:03,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:30:05,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:09,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:10,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:10,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:30:10,620 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:30:10,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:30:10,622 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:30:11,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:30:12,125 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:30:14,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:15,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:18,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:30:20,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:24,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:24,458 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:30:26,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:26,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:30:26,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:30:26,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:30:26,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:30:27,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:30:28,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:30:29,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:31,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:33,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:30:36,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:39,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:41,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:42,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:30:42,901 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:30:42,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:30:42,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:30:43,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:30:44,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:30:44,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:47,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:48,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:30:52,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:54,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:30:54,462 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:30:58,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:30:59,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:30:59,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:30:59,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:30:59,138 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:30:59,165 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:30:59,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:00,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:31:03,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:31:04,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:09,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:09,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:14,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:14,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:15,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:31:15,450 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:31:15,450 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:31:15,452 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:31:16,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:31:16,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:31:18,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:31:19,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:24,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:24,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:24,464 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:31:29,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:29,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:31,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:31:31,395 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:31:31,396 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:31:31,397 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:31:32,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:31:32,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:31:33,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:31:35,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:39,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:40,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:44,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:45,399 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:47,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:31:47,459 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:31:47,460 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:31:47,461 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:31:48,207 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:31:48,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:31:48,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:31:50,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:54,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:31:54,467 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:31:55,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:31:59,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:00,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:03,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:32:03,672 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:32:03,673 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:32:03,674 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:32:03,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:32:04,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:32:04,222 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:32:05,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:09,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:10,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:14,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:15,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:18,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:32:19,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:32:19,620 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:32:19,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:32:19,621 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:32:20,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:32:20,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:32:21,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:24,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:24,470 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:32:27,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:29,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:32,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:33,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:32:35,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:32:35,575 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:32:35,575 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:32:35,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:32:36,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:32:36,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:32:37,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:39,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:42,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:44,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:47,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:48,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:32:51,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:32:51,853 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:32:51,853 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:32:51,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:32:52,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:32:52,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:54,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:32:54,265 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:32:54,474 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:32:58,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:32:59,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:03,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:03,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:33:07,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:33:07,914 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:33:07,915 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:33:07,917 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:33:08,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:33:08,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:09,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:10,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:33:13,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:14,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:18,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:33:18,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:23,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:24,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:33:24,044 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:33:24,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:33:24,046 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:33:24,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:24,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:33:24,477 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:33:26,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:33:29,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:29,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:33,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:33:34,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:39,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:39,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:40,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:33:40,289 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:33:40,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:33:40,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:33:40,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:33:42,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:33:44,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:45,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:48,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:33:50,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:54,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:33:54,480 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:33:55,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:33:56,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:33:56,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:33:56,432 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:33:56,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:33:57,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:33:58,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:33:59,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:01,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:03,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:34:06,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:09,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:11,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:12,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:34:12,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:34:12,617 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:34:12,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:34:13,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:34:14,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:34:14,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:16,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:18,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:34:21,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:24,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:24,484 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:34:27,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:28,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:34:28,699 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:34:28,700 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:34:28,701 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:34:29,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:34:29,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:30,347 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:34:32,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:33,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:34:37,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:39,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:42,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:44,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:45,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:34:45,018 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:34:45,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:34:45,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:34:45,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:34:46,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:34:48,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:48,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:34:53,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:54,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:34:54,486 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:34:58,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:34:59,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:00,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:35:00,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:35:00,984 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:35:00,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:35:01,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:35:02,375 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:35:03,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:35:03,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:08,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:09,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:13,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:14,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:17,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:35:17,128 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:35:17,128 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:35:17,130 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:35:17,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:35:18,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:35:18,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:35:19,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:24,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:24,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:24,488 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:35:29,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:29,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:33,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:35:33,211 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:35:33,212 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:35:33,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:35:33,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:35:33,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:35:34,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:35:35,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:39,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:40,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:44,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:45,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:48,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:35:49,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:35:49,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:35:49,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:35:49,258 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:35:49,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:35:50,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:50,418 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:35:54,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:35:54,491 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:35:55,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:35:59,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:00,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:03,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:36:05,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:36:05,203 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:36:05,204 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:36:05,205 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:36:05,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:36:06,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:06,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:36:09,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:11,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:14,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:16,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:18,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:36:21,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:36:21,221 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:36:21,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:36:21,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:36:21,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:21,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:36:22,444 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:36:24,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:24,495 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:36:26,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:29,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:31,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:33,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:36:36,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:37,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:36:37,183 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:36:37,183 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:36:37,184 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:36:37,459 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:36:38,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:36:39,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:42,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:44,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:47,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:48,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:36:52,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:53,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:36:53,232 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:36:53,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:36:53,233 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:36:53,472 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:36:54,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:36:54,473 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:36:54,498 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:36:57,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:36:58,475 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:36:59,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:00,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:03,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:03,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:37:04,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:08,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:09,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:09,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:10,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:12,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:37:14,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:14,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:14,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:18,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:37:19,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:24,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:24,501 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:37:25,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:29,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:37:29,019 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:37:29,020 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:37:29,021 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:37:29,277 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:29,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:37:30,511 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:31,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:33,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:37:36,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:39,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:41,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:44,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:45,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:37:45,488 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:37:45,488 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:37:45,490 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:37:45,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:37:46,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:46,525 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:37:48,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:37:51,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:54,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:37:54,504 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:37:56,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:37:59,281 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:01,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:01,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:38:01,620 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:38:01,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:38:01,622 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:38:02,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:38:02,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:38:03,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:38:06,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:09,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:11,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:14,284 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:16,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:17,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:38:17,862 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:38:17,863 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:38:17,864 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:38:18,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:38:18,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:38:18,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:38:21,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:24,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:24,508 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:38:27,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:29,286 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:32,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:33,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:38:33,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:38:33,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:38:33,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:38:33,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:38:34,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:38:34,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:38:38,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:39,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:43,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:44,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:48,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:48,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:38:50,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:38:50,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:38:50,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:38:50,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:38:50,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:38:50,583 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:38:53,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:54,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:38:54,512 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:38:58,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:38:59,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:03,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:03,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:39:06,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:39:06,051 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:39:06,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:39:06,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:39:06,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:39:06,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:39:09,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:09,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:14,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:14,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:18,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:39:19,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:22,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:39:22,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:39:22,157 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:39:22,159 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:39:22,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:39:22,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:39:24,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:24,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:24,515 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:39:29,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:29,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:33,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:39:34,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:38,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:39:38,126 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:39:38,127 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:39:38,128 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:39:38,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:39:38,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:39:39,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:40,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:44,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:45,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:48,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:39:50,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:54,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:39:54,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:39:54,090 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:39:54,091 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:39:54,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:39:54,517 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:39:54,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:39:54,642 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:39:55,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:39:59,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:00,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:03,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:40:05,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:09,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:10,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:40:10,034 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:40:10,034 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:40:10,036 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:40:10,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:40:10,656 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:40:11,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:14,311 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:16,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:18,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:40:21,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:24,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:24,520 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:40:25,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:40:26,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:40:26,002 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:40:26,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:40:26,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:40:26,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:40:27,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:29,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:32,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:33,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:40:37,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:39,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:41,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:40:41,850 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:40:41,851 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:40:41,852 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:40:42,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:40:42,683 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:40:42,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:44,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:47,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:48,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:40:52,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:54,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:40:54,525 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:40:58,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:40:58,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:40:58,027 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:40:58,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:40:58,028 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:40:58,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:40:58,697 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:40:59,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:03,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:03,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:41:08,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:09,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:13,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:13,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:41:13,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:41:13,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:41:13,871 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:41:14,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:14,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:41:14,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:41:18,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:41:18,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:23,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:24,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:24,525 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:41:29,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:29,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:30,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:41:30,036 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:41:30,036 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:41:30,038 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:41:30,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:41:30,726 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:41:33,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:41:35,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:39,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:40,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:44,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:45,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:46,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:41:46,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:41:46,106 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:41:46,107 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:41:46,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:41:46,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:41:48,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:41:50,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:54,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:41:54,529 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:41:55,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:41:59,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:00,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:02,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:42:02,176 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:42:02,177 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:42:02,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:42:02,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:42:02,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:42:03,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:42:06,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:09,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:11,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:14,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:16,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:18,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:42:18,042 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:42:18,042 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:42:18,043 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:42:18,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:42:18,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:42:18,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:42:22,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:24,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:24,533 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:42:27,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:29,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:32,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:33,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:42:34,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:42:34,090 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:42:34,090 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:42:34,092 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:42:34,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:42:34,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:42:38,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:39,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:43,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:44,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:48,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:48,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:42:50,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:42:50,128 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:42:50,129 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:42:50,130 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:42:50,796 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:42:50,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:42:53,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:54,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:42:54,538 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:42:58,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:42:59,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:03,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:03,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:43:06,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:43:06,181 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:43:06,181 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:43:06,183 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:43:06,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:43:06,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:43:09,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:09,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:14,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:14,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:18,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:43:19,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:22,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:43:22,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:43:22,132 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:43:22,133 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:43:22,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:43:22,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:43:24,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:24,540 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:43:24,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:29,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:29,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:33,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:43:34,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:38,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:43:38,490 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:43:38,490 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:43:38,491 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:43:38,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:43:38,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:43:39,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:40,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:44,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:45,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:48,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:43:50,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:54,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:43:54,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:43:54,454 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:43:54,455 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:43:54,456 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:43:54,543 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:43:54,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:43:54,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:43:55,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:43:59,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:00,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:03,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:44:05,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:09,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:10,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:10,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:44:10,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:44:10,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:44:10,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:44:10,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:44:10,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:44:14,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:15,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:18,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:44:20,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:24,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:24,548 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:44:26,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:26,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:44:26,708 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:44:26,709 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:44:26,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:44:26,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:44:28,881 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:44:29,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:31,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:33,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:44:36,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:39,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:41,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:42,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:44:42,759 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:44:42,760 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:44:42,761 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:44:42,895 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:44:44,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:45,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:44:46,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:48,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:44:51,763 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:54,354 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:44:54,550 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:44:57,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:44:58,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:44:58,904 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:44:58,904 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:44:58,906 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:44:58,909 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:44:59,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:01,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:45:02,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:03,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:45:07,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:09,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:12,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:14,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:15,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:45:15,135 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:45:15,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:45:15,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:45:15,924 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:45:17,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:45:18,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:18,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:45:23,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:24,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:24,553 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:45:28,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:29,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:31,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:45:31,297 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:45:31,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:45:31,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:45:31,937 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:45:33,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:45:33,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:45:34,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:39,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:39,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:44,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:44,362 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:47,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:45:47,260 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:45:47,261 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:45:47,262 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:45:47,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:45:48,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:45:49,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:45:50,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:54,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:45:54,555 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:45:55,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:45:59,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:00,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:03,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:46:03,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:46:03,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:46:03,870 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:46:03,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:46:03,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:46:05,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:05,967 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:46:09,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:10,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:14,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:15,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:18,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:46:19,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:46:19,840 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:46:19,840 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:46:19,842 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:46:19,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:46:21,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:21,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:46:24,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:24,558 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:46:27,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:29,369 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:32,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:33,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:46:36,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:46:36,201 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:46:36,201 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:46:36,202 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:46:36,998 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:46:37,999 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:46:38,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:39,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:43,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:44,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:48,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:48,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:46:52,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:46:52,552 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:46:52,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:46:52,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:46:53,012 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:46:53,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:46:54,013 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:46:54,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:54,560 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:46:59,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:46:59,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:03,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:47:04,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:08,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:47:08,825 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:47:08,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:47:08,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:47:09,026 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:47:09,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:09,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:12,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:47:14,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:14,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:18,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:47:19,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:24,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:24,564 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:47:25,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:47:25,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:47:25,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:47:25,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:25,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:47:25,040 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:47:28,043 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:47:29,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:30,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:33,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:47:35,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:39,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:40,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:41,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:47:41,253 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:47:41,253 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:47:41,254 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:47:42,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:47:44,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:47:44,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:45,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:48,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:47:50,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:54,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:47:54,569 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:47:55,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:47:57,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:47:57,485 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:47:57,485 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:47:57,487 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:47:58,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:47:59,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:00,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:48:01,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:03,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:48:06,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:09,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:11,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:13,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:48:13,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:48:13,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:48:13,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:48:14,082 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:48:14,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:16,084 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:48:16,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:18,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:48:21,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:24,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:24,571 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:48:27,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:29,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:29,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:48:29,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:48:29,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:48:29,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:48:30,097 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:48:32,098 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:48:32,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:33,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:48:37,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:39,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:42,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:44,389 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:45,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:48:45,767 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:48:45,767 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:48:45,769 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:48:46,110 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:48:47,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:48,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:48:48,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:48:52,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:54,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:48:54,574 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:48:58,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:48:59,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:01,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:49:01,898 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:49:01,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:49:01,901 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:49:02,124 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:49:03,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:49:03,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:04,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:49:08,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:09,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:13,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:14,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:17,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:49:17,829 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:49:17,829 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:49:17,831 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:49:18,138 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:49:18,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:49:19,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:20,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:49:24,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:24,578 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:49:25,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:29,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:30,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:33,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:49:34,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:49:34,061 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:49:34,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:49:34,064 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:49:34,152 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:49:36,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:36,153 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:49:39,397 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:41,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:44,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:46,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:48,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:49:50,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:49:50,011 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:49:50,011 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:49:50,013 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:49:50,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:49:52,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:52,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:49:54,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:49:54,581 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:49:57,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:49:59,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:02,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:03,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:50:05,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:50:05,976 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:50:05,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:50:05,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:50:06,181 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:50:08,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:08,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:50:09,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:13,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:14,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:18,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:18,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:50:22,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:50:22,025 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:50:22,025 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:50:22,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:50:22,194 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:50:23,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:24,196 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:50:24,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:24,583 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:50:28,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:29,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:33,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:33,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:50:38,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:50:38,074 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:50:38,074 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:50:38,075 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:50:38,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:50:39,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:39,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:40,209 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:50:44,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:44,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:48,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:50:49,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:54,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:50:54,031 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:50:54,032 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:50:54,033 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:50:54,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:50:54,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:54,619 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:50:54,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:50:56,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:50:59,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:50:59,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:03,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:51:04,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:09,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:09,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:09,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:51:09,992 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:51:09,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:51:09,994 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:51:10,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:51:12,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:51:14,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:14,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:18,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:51:19,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:24,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:24,623 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:51:25,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:25,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:51:25,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:51:25,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:51:25,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:51:26,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:51:28,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:51:29,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:30,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:33,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:51:35,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:39,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:40,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:42,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:51:42,083 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:51:42,083 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:51:42,085 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:51:42,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:51:44,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:51:44,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:46,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:48,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:51:51,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:54,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:51:54,626 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:51:56,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:51:58,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:51:58,159 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:51:58,159 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:51:58,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:51:58,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:51:59,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:00,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:52:02,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:03,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:52:07,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:09,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:12,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:14,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:52:14,007 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:52:14,007 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:52:14,009 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:52:14,295 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:52:14,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:16,297 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:52:18,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:18,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:52:23,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:24,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:24,628 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:52:28,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:29,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:29,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:52:29,971 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:52:29,972 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:52:29,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:52:30,307 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:52:32,308 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:52:33,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:52:33,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:38,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:39,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:43,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:44,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:46,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:52:46,014 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:52:46,015 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:52:46,016 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:52:46,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:52:48,320 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:52:48,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:52:49,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:54,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:52:54,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:54,632 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:52:59,520 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:52:59,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:02,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:53:02,196 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:53:02,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:53:02,198 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:53:02,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:53:03,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:53:04,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:53:05,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:09,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:10,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:14,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:15,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:18,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:53:18,255 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:53:18,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:53:18,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:53:18,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:53:18,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:53:20,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:20,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:53:24,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:24,634 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:53:25,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:29,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:30,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:33,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:53:34,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:53:34,295 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:53:34,295 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:53:34,297 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:53:34,354 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:53:36,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:36,356 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:53:39,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:41,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:44,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:46,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:48,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:53:50,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:53:50,418 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:53:50,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:53:50,419 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:53:51,367 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:53:51,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:52,368 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:53:54,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:53:54,637 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:53:56,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:53:59,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:01,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:03,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:54:06,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:54:06,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:54:06,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:54:06,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:54:07,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:54:07,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:08,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:54:09,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:12,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:14,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:17,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:18,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:54:22,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:22,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:54:22,615 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:54:22,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:54:22,617 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:54:23,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:54:24,391 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:54:24,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:24,640 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:54:27,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:29,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:32,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:33,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:54:37,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:38,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:54:38,744 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:54:38,744 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:54:38,746 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:54:39,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:54:39,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:40,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:54:42,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:44,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:47,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:48,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:54:52,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:54,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:54:54,645 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:54:54,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:54:54,794 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:54:54,794 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:54:54,795 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:54:55,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:54:56,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:54:57,796 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:54:59,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:02,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:03,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:55:07,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:09,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:10,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:55:10,827 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:55:10,827 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:55:10,829 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:55:11,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:55:12,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:55:12,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:14,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:17,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:18,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:55:22,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:24,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:24,646 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:55:27,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:55:27,253 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:55:27,254 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:55:27,255 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:55:27,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:55:28,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:28,438 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:55:29,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:33,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:33,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:55:38,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:39,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:43,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:43,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:55:43,290 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:55:43,290 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:55:43,292 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:55:43,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:55:44,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:55:44,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:48,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:48,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:55:53,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:54,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:55:54,649 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:55:58,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:55:59,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:55:59,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:55:59,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:55:59,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:55:59,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:55:59,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:00,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:56:03,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:56:04,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:09,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:09,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:14,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:14,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:15,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:56:15,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:56:15,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:56:15,760 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:56:16,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:56:18,483 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:56:18,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:56:19,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:24,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:24,654 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:56:25,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:29,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:30,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:31,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:56:31,711 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:56:31,711 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:56:31,712 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:56:32,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:56:33,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:56:34,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:56:35,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:39,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:40,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:44,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:45,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:48,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:56:48,031 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:56:48,031 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:56:48,033 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:56:48,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:56:48,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:56:50,507 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:56:51,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:54,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:56:54,655 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:56:56,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:56:59,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:01,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:03,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:57:04,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:57:04,326 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:57:04,327 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:57:04,328 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:57:04,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:57:06,520 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:57:07,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:09,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:12,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:14,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:17,368 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:18,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:57:20,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:57:20,537 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:57:20,537 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:57:20,539 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:57:21,533 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:57:22,534 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:57:22,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:24,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:24,659 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:57:27,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:29,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:32,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:33,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:57:36,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:57:36,781 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:57:36,781 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:57:36,782 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:57:37,545 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:57:37,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:38,546 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:57:39,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:42,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:44,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:47,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:48,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:57:52,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:52,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:57:52,985 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:57:52,985 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:57:52,987 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:57:53,557 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:57:54,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:57:54,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:57:54,660 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:57:58,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:57:59,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:03,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:03,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:58:08,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:09,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:58:09,444 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:58:09,444 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:58:09,446 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:58:09,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:58:09,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:10,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:58:14,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:14,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:18,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:58:19,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:24,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:24,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:24,664 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:58:25,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:58:25,582 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:58:25,583 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:58:25,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:58:25,584 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:58:28,587 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:58:29,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:29,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:33,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:58:34,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:39,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:39,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:41,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:58:41,710 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:58:41,711 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:58:41,712 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:58:42,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:58:44,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:44,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:58:44,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:48,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:58:49,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:54,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:58:54,667 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:58:55,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:58:57,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:58:57,749 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:58:57,749 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:58:57,751 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:58:58,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:58:59,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:00,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:59:00,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:03,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:59:05,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:09,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:10,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:13,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:59:13,975 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:59:13,976 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:59:13,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:59:14,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:14,623 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:59:15,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:16,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:59:18,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:59:20,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:24,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:24,670 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:59:26,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:29,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:29,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:59:29,898 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:59:29,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:59:29,899 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:59:30,635 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:59:31,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:32,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:59:33,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:59:36,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:39,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:41,901 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:44,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:46,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 14:59:46,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 14:59:46,046 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 14:59:46,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 14:59:46,648 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 14:59:47,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:48,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 14:59:48,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 14:59:52,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:54,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 14:59:54,675 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 14:59:57,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 14:59:59,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:01,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:00:01,901 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:00:01,901 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:00:01,902 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:00:02,662 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:00:02,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:03,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:00:04,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:00:07,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:09,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:12,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:14,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:17,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:18,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:00:18,048 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:00:18,048 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:00:18,050 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:00:18,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:00:18,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:00:20,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:00:23,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:24,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:24,677 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:00:28,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:29,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:33,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:33,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:00:34,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:00:34,035 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:00:34,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:00:34,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:00:34,689 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:00:36,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:00:39,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:39,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:44,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:44,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:48,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:00:49,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:49,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:00:50,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:00:50,002 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:00:50,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:00:50,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:00:52,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:00:54,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:54,679 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:00:54,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:00:59,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:00:59,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:03,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:01:04,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:05,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:01:05,948 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:01:05,949 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:01:05,950 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:01:06,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:01:08,717 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:01:09,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:09,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:14,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:14,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:18,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:01:19,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:21,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:01:21,897 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:01:21,898 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:01:21,899 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:01:22,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:01:24,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:24,684 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:01:24,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:01:25,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:29,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:30,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:33,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:01:35,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:37,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:01:37,838 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:01:37,839 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:01:37,840 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:01:38,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:01:39,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:40,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:01:40,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:44,603 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:45,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:48,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:01:50,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:54,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:01:54,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:01:54,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:01:54,006 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:01:54,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:01:54,688 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:01:54,753 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:01:56,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:01:56,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:01:59,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:01,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:03,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:02:06,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:09,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:10,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:02:10,156 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:02:10,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:02:10,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:02:10,766 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:02:12,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:12,767 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:02:14,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:17,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:18,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:02:22,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:24,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:24,691 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:02:26,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:02:26,004 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:02:26,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:02:26,006 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:02:26,779 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:02:28,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:28,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:02:29,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:33,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:33,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:02:38,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:39,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:42,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:02:42,134 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:02:42,134 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:02:42,136 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:02:42,794 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:02:43,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:44,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:44,795 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:02:48,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:48,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:02:53,137 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:54,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:02:54,694 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:02:58,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:02:58,175 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:02:58,176 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:02:58,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:02:58,178 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:02:58,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:02:59,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:00,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:03:03,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:03,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:03:08,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:09,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:13,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:14,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:03:14,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:03:14,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:03:14,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:03:14,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:14,823 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:03:16,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:03:18,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:18,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:03:23,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:24,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:24,697 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:03:28,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:29,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:30,282 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:03:30,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:03:30,285 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:03:30,287 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:03:30,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:03:32,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:03:33,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:03:34,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:39,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:39,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:44,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:44,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:46,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:03:46,324 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:03:46,324 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:03:46,325 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:03:46,850 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:03:48,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:03:48,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:03:49,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:54,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:03:54,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:54,701 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:03:59,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:03:59,702 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:02,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:04:02,546 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:04:02,546 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:04:02,548 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:04:02,863 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:04:03,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:04:04,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:04:05,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:09,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:10,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:14,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:15,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:18,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:04:19,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:04:19,072 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:04:19,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:04:19,074 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:04:19,876 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:04:20,877 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:04:21,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:24,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:24,704 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:04:26,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:29,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:31,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:33,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:04:35,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:04:35,200 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:04:35,200 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:04:35,201 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:04:35,888 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:04:36,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:04:37,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:39,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:42,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:44,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:47,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:48,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:04:51,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:04:51,233 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:04:51,233 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:04:51,234 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:04:51,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:04:52,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:52,902 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:04:54,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:04:54,709 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:04:57,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:04:59,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:02,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:03,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:05:07,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:05:07,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:05:07,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:05:07,276 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:05:07,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:05:08,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:08,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:05:09,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:13,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:14,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:18,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:18,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:05:23,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:05:23,219 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:05:23,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:05:23,221 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:05:23,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:05:24,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:24,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:24,710 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:05:24,931 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:05:29,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:29,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:33,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:05:34,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:39,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:39,571 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:05:39,574 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:05:39,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:05:39,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:05:39,944 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:05:40,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:40,945 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:05:44,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:45,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:48,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:05:50,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:54,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:05:54,713 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:05:55,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:05:55,626 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:05:55,626 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:05:55,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:05:55,628 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:05:55,956 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:05:56,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:05:59,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:00,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:03,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:06:05,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:09,025 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:10,630 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:11,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:06:11,575 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:06:11,575 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:06:11,577 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:06:11,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:06:12,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:06:14,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:16,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:18,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:06:21,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:24,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:24,717 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:06:26,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:28,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:06:28,038 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:06:28,039 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:06:28,041 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:06:28,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:06:28,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:06:29,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:32,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:33,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:06:37,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:39,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:42,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:44,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:44,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:06:44,078 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:06:44,078 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:06:44,080 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:06:44,994 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:06:46,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:06:47,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:48,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:06:52,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:54,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:06:54,720 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:06:57,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:06:59,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:00,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:07:00,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:07:00,353 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:07:00,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:07:01,007 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:07:03,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:07:03,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:03,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:07:08,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:09,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:13,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:14,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:16,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:07:16,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:07:16,541 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:07:16,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:07:17,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:07:18,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:18,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:07:19,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:07:23,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:24,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:24,723 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:07:28,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:29,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:32,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:07:32,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:07:32,891 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:07:32,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:07:33,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:07:33,707 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:07:33,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:35,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:07:38,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:39,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:43,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:44,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:48,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:07:48,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:49,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:07:49,147 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:07:49,147 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:07:49,150 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:07:50,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:07:51,049 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:07:54,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:54,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:07:54,726 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:07:59,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:07:59,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:03,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:08:04,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:05,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:08:05,351 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:08:05,351 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:08:05,353 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:08:06,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:08:07,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:08:09,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:10,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:14,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:15,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:18,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:08:20,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:21,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:08:21,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:08:21,556 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:08:21,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:08:22,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:08:23,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:08:24,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:24,730 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:08:25,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:29,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:30,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:33,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:08:35,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:37,706 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:08:37,709 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:08:37,709 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:08:37,710 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:08:38,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:08:39,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:39,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:08:40,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:44,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:45,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:48,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:08:50,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:53,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:08:53,922 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:08:53,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:08:53,924 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:08:54,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:08:54,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:08:54,732 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:08:55,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:08:56,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:08:59,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:01,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:03,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:09:06,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:09,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:09,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:09:09,953 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:09:09,953 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:09:09,955 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:09:10,114 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:09:11,114 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:09:11,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:14,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:16,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:18,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:09:21,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:24,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:24,735 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:09:26,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:09:26,110 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:09:26,111 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:09:26,112 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:09:26,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:09:27,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:27,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:09:29,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:32,113 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:33,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:09:37,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:39,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:42,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:42,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:09:42,148 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:09:42,148 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:09:42,149 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:09:43,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:09:43,145 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:09:44,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:47,150 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:48,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:09:52,151 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:54,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:54,738 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:09:57,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:09:58,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:09:58,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:09:58,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:09:58,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:09:59,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:09:59,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:10:01,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:10:03,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:03,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:10:08,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:09,063 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:13,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:14,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:14,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:10:14,258 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:10:14,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:10:14,260 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:10:15,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:10:15,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:10:18,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:10:19,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:24,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:24,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:24,741 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:10:29,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:29,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:30,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:10:30,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:10:30,362 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:10:30,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:10:31,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:10:33,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:10:33,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:10:35,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:39,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:40,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:44,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:45,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:46,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:10:46,507 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:10:46,507 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:10:46,509 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:10:47,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:10:48,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:10:49,198 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:10:50,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:54,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:10:54,744 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:10:55,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:10:59,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:00,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:02,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:11:02,458 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:11:02,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:11:02,460 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:11:03,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:11:03,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:11:05,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:11:06,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:09,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:11,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:14,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:16,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:18,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:11:18,529 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:11:18,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:11:18,530 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:11:18,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:11:19,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:11:21,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:11:21,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:24,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:24,748 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:11:26,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:29,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:31,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:33,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:11:34,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:11:34,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:11:34,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:11:34,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:11:35,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:11:37,237 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:11:37,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:39,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:42,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:44,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:47,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:48,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:11:50,425 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:11:50,427 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:11:50,428 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:11:50,429 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:11:51,248 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:11:53,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:11:53,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:54,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:11:54,750 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:11:58,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:11:59,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:03,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:12:03,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:06,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:12:06,594 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:12:06,595 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:12:06,596 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:12:07,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:12:09,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:09,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:12:09,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:14,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:14,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:18,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:12:19,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:22,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:12:22,740 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:12:22,741 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:12:22,742 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:12:23,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:12:24,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:24,754 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:12:24,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:25,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:12:29,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:29,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:33,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:12:34,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:38,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:12:38,682 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:12:38,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:12:38,684 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:12:39,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:39,286 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:12:40,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:41,287 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:12:44,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:45,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:48,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:12:50,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:54,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:12:54,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:12:54,551 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:12:54,552 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:12:54,553 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:12:54,756 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:12:55,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:12:55,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:12:57,300 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:12:59,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:00,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:03,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:13:05,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:09,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:10,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:13:10,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:13:10,684 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:13:10,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:13:11,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:13:11,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:13,313 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:13:14,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:16,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:18,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:13:21,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:24,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:24,761 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:13:26,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:13:26,632 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:13:26,632 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:13:26,634 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:13:27,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:13:27,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:29,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:29,326 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:13:32,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:33,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:13:37,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:39,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:42,637 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:42,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:13:42,815 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:13:42,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:13:42,816 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:13:43,339 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:13:44,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:45,341 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:13:47,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:48,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:13:52,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:54,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:54,763 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:13:58,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:13:58,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:13:58,789 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:13:58,790 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:13:58,792 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:13:59,098 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:13:59,352 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:14:01,353 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:14:03,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:14:03,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:08,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:09,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:13,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:14,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:14,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:14:14,944 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:14:14,944 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:14:14,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:14:15,365 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:14:17,366 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:14:18,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:14:18,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:23,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:24,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:24,766 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:14:29,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:29,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:30,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:14:30,892 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:14:30,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:14:30,894 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:14:31,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:14:33,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:14:33,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:14:34,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:39,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:39,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:44,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:44,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:46,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:14:46,930 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:14:46,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:14:46,932 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:14:47,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:14:48,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:14:49,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:14:49,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:54,106 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:14:54,774 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:14:55,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:14:59,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:00,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:02,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:15:02,981 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:15:02,982 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:15:02,983 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:15:03,403 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:15:03,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:15:05,404 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:15:05,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:09,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:10,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:14,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:15,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:18,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:15:19,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:15:19,256 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:15:19,256 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:15:19,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:15:19,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:15:21,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:21,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:15:24,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:24,777 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:15:26,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:29,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:31,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:33,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:15:35,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:15:35,206 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:15:35,206 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:15:35,208 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:15:35,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:15:37,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:37,431 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:15:39,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:42,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:44,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:47,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:48,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:15:51,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:15:51,444 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:15:51,444 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:15:51,445 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:15:52,442 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:15:52,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:53,443 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:15:54,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:15:54,780 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:15:57,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:15:59,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:02,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:03,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:16:07,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:16:07,500 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:16:07,500 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:16:07,502 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:16:08,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:16:08,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:09,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:09,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:16:13,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:14,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:18,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:18,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:16:23,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:23,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:16:23,548 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:16:23,548 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:16:23,550 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:16:24,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:24,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:16:24,783 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:16:25,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:16:28,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:29,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:33,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:16:33,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:38,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:39,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:40,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:16:40,017 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:16:40,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:16:40,019 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:16:40,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:16:41,480 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:16:44,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:44,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:48,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:16:49,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:54,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:16:54,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:54,788 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:16:55,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:16:55,954 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:16:55,955 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:16:55,956 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:16:56,491 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:16:57,492 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:16:59,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:16:59,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:03,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:17:04,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:09,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:09,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:12,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:17:12,190 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:17:12,190 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:17:12,191 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:17:12,505 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:17:13,506 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:17:14,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:15,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:18,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:17:20,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:24,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:24,790 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:17:25,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:28,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:17:28,407 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:17:28,407 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:17:28,409 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:17:28,518 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:17:29,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:29,519 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:17:31,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:33,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:17:36,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:39,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:41,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:44,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:44,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:17:44,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:17:44,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:17:44,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:17:45,532 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:17:45,533 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:17:46,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:48,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:17:51,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:54,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:17:54,793 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:17:56,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:17:59,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:00,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:18:00,845 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:18:00,846 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:18:00,847 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:18:01,546 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:18:01,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:03,548 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:18:03,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:18:06,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:09,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:11,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:14,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:16,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:17,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:18:17,068 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:18:17,068 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:18:17,070 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:18:17,559 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:18:18,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:18:19,560 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:18:22,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:24,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:24,797 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:18:27,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:29,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:32,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:33,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:18:33,376 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:18:33,377 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:18:33,379 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:18:33,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:18:33,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:18:35,573 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:18:38,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:39,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:43,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:44,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:48,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:48,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:18:49,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:18:49,597 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:18:49,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:18:49,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:18:50,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:18:51,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:18:53,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:54,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:18:54,799 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:18:58,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:18:59,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:03,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:19:03,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:05,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:19:05,649 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:19:05,649 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:19:05,651 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:19:06,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:19:07,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:19:09,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:09,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:14,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:14,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:18,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:19:19,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:21,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:19:21,882 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:19:21,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:19:21,884 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:19:22,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:19:23,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:19:24,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:24,802 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:19:24,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:29,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:29,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:33,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:19:34,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:37,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:19:37,890 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:19:37,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:19:37,892 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:19:38,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:19:39,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:39,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:19:39,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:44,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:44,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:19:49,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:54,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:19:54,035 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:19:54,035 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:19:54,037 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:19:54,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:19:54,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:19:54,805 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:19:55,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:19:55,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:19:59,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:00,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:03,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:20:05,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:09,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:10,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:20:10,072 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:20:10,073 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:20:10,074 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:20:10,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:20:11,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:11,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:20:14,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:16,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:18,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:20:21,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:24,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:24,810 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:20:26,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:20:26,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:20:26,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:20:26,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:26,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:20:26,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:20:27,664 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:20:29,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:31,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:33,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:20:36,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:39,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:41,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:42,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:20:42,195 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:20:42,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:20:42,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:20:42,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:20:43,677 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:20:44,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:46,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:20:51,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:54,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:54,811 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:20:56,812 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:20:58,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:20:58,249 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:20:58,249 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:20:58,250 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:20:58,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:20:59,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:20:59,691 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:21:02,251 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:03,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:21:07,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:09,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:12,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:14,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:14,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:21:14,292 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:21:14,292 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:21:14,293 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:21:14,703 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:21:15,704 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:21:17,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:18,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:21:22,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:24,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:24,814 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:21:27,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:29,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:30,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:21:30,243 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:21:30,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:21:30,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:21:30,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:21:31,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:21:33,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:33,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:21:38,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:39,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:43,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:44,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:46,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:21:46,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:21:46,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:21:46,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:21:46,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:21:47,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:21:48,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:21:53,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:54,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:21:54,818 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:21:58,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:21:59,297 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:02,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:22:02,408 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:22:02,408 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:22:02,410 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:22:02,746 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:22:03,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:22:03,747 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:22:04,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:09,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:09,411 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:14,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:14,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:18,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:22:18,469 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:22:18,469 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:22:18,471 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:22:18,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:22:18,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:22:19,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:19,759 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:22:24,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:24,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:24,820 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:22:29,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:29,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:33,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:22:34,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:22:34,643 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:22:34,643 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:22:34,645 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:22:34,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:22:35,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:35,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:22:39,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:40,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:44,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:45,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:48,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:22:50,615 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:22:50,617 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:22:50,618 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:22:50,619 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:22:50,783 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:22:51,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:51,784 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:22:54,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:22:54,823 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:22:56,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:22:59,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:01,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:03,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:23:06,454 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:23:06,456 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:23:06,457 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:23:06,458 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:23:06,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:23:07,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:07,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:23:09,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:12,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:14,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:17,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:18,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:23:22,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:23:22,505 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:23:22,505 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:23:22,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:22,507 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:23:22,809 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:23:23,810 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:23:24,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:24,826 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:23:27,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:29,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:32,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:33,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:23:37,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:38,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:23:38,538 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:23:38,538 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:23:38,539 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:23:38,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:23:39,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:39,821 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:23:43,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:44,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:48,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:48,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:23:53,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:54,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:23:54,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:23:54,724 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:23:54,724 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:23:54,726 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:23:54,829 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:23:54,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:23:55,834 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:23:58,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:23:59,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:03,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:24:03,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:08,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:09,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:10,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:24:10,763 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:24:10,764 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:24:10,766 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:24:10,848 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:24:11,849 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:24:14,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:14,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:18,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:24:19,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:24,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:24,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:24,832 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:24:26,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:24:26,822 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:24:26,822 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:24:26,824 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:24:26,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:24:27,862 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:24:29,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:29,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:33,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:24:34,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:39,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:39,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:43,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:24:43,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:24:43,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:24:43,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:24:43,875 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:24:43,876 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:24:44,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:45,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:48,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:24:50,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:54,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:54,835 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:24:55,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:24:59,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:24:59,115 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:24:59,115 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:24:59,116 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:24:59,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:24:59,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:24:59,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:25:01,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:03,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:25:06,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:09,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:11,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:14,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:15,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:25:15,265 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:25:15,265 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:25:15,267 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:25:15,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:25:15,901 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:25:16,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:18,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:25:21,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:24,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:24,841 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:25:26,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:29,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:31,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:25:31,424 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:25:31,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:25:31,426 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:25:31,913 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:25:32,427 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:33,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:25:33,915 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:25:37,428 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:39,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:42,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:44,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:47,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:47,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:25:47,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:25:47,466 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:25:47,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:25:47,926 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:25:48,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:25:49,928 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:25:52,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:54,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:25:54,879 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:25:57,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:25:59,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:02,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:03,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:26:03,624 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:26:03,624 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:26:03,626 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:26:03,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:26:03,939 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:26:05,940 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:26:08,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:09,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:13,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:14,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:18,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:18,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:26:19,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:26:19,821 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:26:19,821 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:26:19,822 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:26:19,952 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:26:21,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:26:23,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:24,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:24,881 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:26:28,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:29,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:33,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:26:33,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:36,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:26:36,044 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:26:36,044 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:26:36,045 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:26:36,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:26:37,966 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:26:39,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:39,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:44,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:44,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:48,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:26:49,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:52,287 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:26:52,289 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:26:52,289 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:26:52,291 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:26:52,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:26:53,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:26:54,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:26:54,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:54,885 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:26:59,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:26:59,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:03,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:27:04,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:08,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:27:08,421 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:27:08,421 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:27:08,423 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:27:08,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:27:09,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:09,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:27:10,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:14,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:15,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:18,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:27:20,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:24,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:24,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:27:24,688 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:27:24,689 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:27:24,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:27:24,888 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:27:25,002 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:27:25,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:26,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:27:29,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:30,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:33,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:27:35,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:39,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:40,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:27:40,730 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:27:40,730 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:27:40,732 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:27:41,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:27:41,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:42,015 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:27:44,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:46,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:48,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:27:51,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:54,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:27:54,891 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:27:56,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:27:57,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:27:57,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:27:57,037 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:27:57,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:27:58,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:27:58,029 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:27:59,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:02,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:03,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:28:07,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:09,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:12,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:13,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:28:13,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:28:13,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:28:13,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:28:14,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:28:14,042 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:28:14,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:17,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:18,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:28:22,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:24,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:24,894 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:28:27,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:29,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:28:29,208 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:28:29,209 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:28:29,210 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:28:29,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:30,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:28:30,054 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:28:33,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:33,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:28:38,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:39,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:43,213 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:44,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:45,395 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:28:45,397 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:28:45,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:28:45,399 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:28:46,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:28:46,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:28:48,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:48,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:28:53,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:54,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:28:54,899 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:28:58,900 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:28:59,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:01,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:29:01,654 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:29:01,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:29:01,656 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:29:02,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:29:03,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:29:04,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:29:04,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:09,409 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:09,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:14,410 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:14,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:17,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:29:17,788 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:29:17,788 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:29:17,790 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:29:18,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:29:18,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:29:19,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:20,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:29:24,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:24,791 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:24,900 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:29:29,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:29,902 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:33,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:29:33,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:29:33,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:29:33,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:29:33,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:29:34,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:29:35,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:36,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:29:39,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:40,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:44,415 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:45,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:48,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:29:49,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:29:49,864 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:29:49,864 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:29:49,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:29:50,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:29:50,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:52,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:29:54,416 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:29:54,903 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:29:56,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:29:59,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:01,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:03,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:30:05,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:30:05,727 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:30:05,728 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:30:05,729 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:30:06,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:30:07,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:08,135 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:30:09,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:12,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:14,419 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:17,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:18,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:30:21,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:30:21,869 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:30:21,869 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:30:21,870 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:30:22,146 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:30:22,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:24,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:30:24,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:24,908 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:30:28,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:29,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:33,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:30:33,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:37,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:30:37,723 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:30:37,723 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:30:37,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:30:38,159 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:30:39,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:39,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:40,160 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:30:44,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:44,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:48,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:30:49,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:53,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:30:53,799 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:30:53,799 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:30:53,801 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:30:54,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:30:54,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:30:54,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:30:54,910 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:30:56,173 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:30:58,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:30:59,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:00,129 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:02,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:03,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:31:04,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:05,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:09,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:10,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:11,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:12,186 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:13,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:31:14,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:16,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:16,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:18,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:31:21,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:24,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:24,913 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:31:27,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:29,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:30,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:31:30,319 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:31:30,319 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:31:30,320 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:31:31,205 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:31:32,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:33,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:33,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:31:38,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:39,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:43,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:44,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:46,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:31:46,656 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:31:46,656 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:31:46,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:31:47,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:31:48,218 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:31:48,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:48,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:31:53,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:54,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:31:54,916 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:31:58,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:31:59,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:02,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:32:02,757 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:32:02,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:32:02,759 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:32:03,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:32:03,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:32:04,233 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:32:04,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:09,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:09,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:14,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:14,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:18,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:32:18,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:32:18,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:32:18,785 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:32:18,786 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:32:19,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:32:19,826 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:20,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:32:24,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:24,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:24,919 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:32:29,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:29,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:33,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:32:34,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:35,099 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:32:35,101 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:32:35,101 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:32:35,103 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:32:35,258 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:32:36,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:32:39,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:40,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:44,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:45,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:48,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:32:50,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:51,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:32:51,024 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:32:51,024 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:32:51,025 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:32:51,271 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:32:52,272 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:32:54,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:32:54,922 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:32:55,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:32:59,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:00,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:03,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:33:05,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:06,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:33:06,945 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:33:06,945 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:33:06,946 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:33:07,284 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:33:08,285 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:33:09,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:10,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:14,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:15,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:18,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:33:20,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:22,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:33:22,976 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:33:22,977 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:33:22,978 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:33:23,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:33:24,298 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:33:24,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:24,925 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:33:26,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:29,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:31,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:33,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:33:36,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:38,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:33:38,834 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:33:38,834 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:33:38,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:33:39,310 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:33:39,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:40,311 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:33:42,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:44,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:47,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:48,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:33:52,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:54,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:33:54,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:33:54,808 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:33:54,809 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:33:54,810 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:33:54,929 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:33:55,323 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:33:56,324 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:33:57,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:33:59,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:02,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:03,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:34:07,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:09,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:10,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:34:10,768 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:34:10,768 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:34:10,770 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:34:11,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:34:12,338 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:34:13,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:14,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:18,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:34:18,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:23,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:24,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:24,931 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:34:26,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:34:26,628 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:34:26,628 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:34:26,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:34:27,350 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:34:28,351 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:34:29,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:29,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:33,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:34:34,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:39,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:39,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:42,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:34:42,685 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:34:42,686 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:34:42,687 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:34:43,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:34:44,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:34:44,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:44,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:48,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:34:49,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:54,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:34:54,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:34:54,934 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:34:58,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:34:58,682 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:34:58,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:34:58,684 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:34:59,376 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:34:59,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:00,377 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:35:00,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:03,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:35:05,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:09,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:10,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:14,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:35:14,549 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:35:14,549 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:35:14,551 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:35:14,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:15,389 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:35:16,390 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:35:16,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:18,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:35:21,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:24,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:24,938 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:35:26,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:29,717 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:30,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:35:30,625 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:35:30,626 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:35:30,627 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:35:31,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:35:32,402 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:35:32,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:33,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:35:37,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:39,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:42,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:44,719 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:46,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:35:46,677 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:35:46,677 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:35:46,679 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:35:47,416 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:35:47,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:48,417 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:35:48,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:35:52,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:54,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:35:54,940 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:35:57,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:35:59,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:02,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:36:02,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:36:02,536 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:36:02,538 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:36:03,429 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:36:03,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:03,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:36:04,430 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:36:08,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:09,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:13,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:14,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:18,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:36:18,594 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:36:18,595 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:36:18,596 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:36:18,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:36:19,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:36:20,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:36:23,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:24,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:24,943 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:36:28,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:29,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:33,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:36:33,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:34,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:36:34,553 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:36:34,553 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:36:34,554 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:36:35,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:36:36,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:36:39,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:39,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:44,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:44,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:48,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:36:49,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:50,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:36:50,616 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:36:50,616 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:36:50,617 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:36:51,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:36:52,475 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:36:53,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:54,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:36:54,947 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:36:58,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:36:59,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:03,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:37:04,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:06,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:37:06,703 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:37:06,703 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:37:06,705 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:37:07,487 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:37:08,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:37:08,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:10,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:13,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:15,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:18,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:37:20,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:22,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:37:22,734 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:37:22,735 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:37:22,736 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:37:23,501 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:37:23,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:24,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:37:24,950 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:37:25,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:28,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:30,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:33,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:37:35,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:38,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:38,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:37:38,784 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:37:38,784 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:37:38,786 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:37:39,513 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:37:40,514 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:37:41,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:43,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:46,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:48,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:37:51,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:53,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:37:54,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:37:54,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:37:54,733 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:37:54,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:37:54,952 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:37:55,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:37:56,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:37:56,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:37:58,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:01,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:03,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:38:06,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:08,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:10,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:38:10,693 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:38:10,693 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:38:10,694 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:38:11,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:38:12,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:38:12,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:13,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:17,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:18,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:38:22,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:23,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:24,955 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:38:27,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:38:27,018 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:38:27,018 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:38:27,020 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:38:27,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:38:28,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:28,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:38:28,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:33,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:33,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:38:38,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:38,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:42,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:38:42,978 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:38:42,978 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:38:42,980 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:38:43,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:38:43,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:43,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:44,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:38:48,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:38:48,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:53,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:53,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:38:54,960 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:38:58,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:38:58,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:38:58,826 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:38:58,826 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:38:58,827 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:38:59,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:38:59,828 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:00,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:39:03,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:39:04,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:08,751 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:09,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:13,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:14,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:15,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:39:15,316 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:39:15,316 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:39:15,317 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:39:15,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:39:16,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:39:18,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:39:20,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:23,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:24,962 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:39:25,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:28,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:30,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:31,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:39:31,288 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:39:31,288 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:39:31,290 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:39:31,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:39:32,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:39:33,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:39:36,291 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:38,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:41,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:43,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:46,292 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:47,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:39:47,429 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:39:47,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:39:47,431 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:39:47,621 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:39:48,622 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:39:48,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:39:51,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:53,757 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:39:54,965 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:39:56,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:39:58,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:01,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:03,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:40:03,712 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:40:03,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:40:03,713 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:40:03,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:40:04,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:40:04,636 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:40:07,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:08,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:12,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:13,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:17,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:18,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:40:19,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:40:19,843 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:40:19,843 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:40:19,844 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:40:20,649 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:40:20,650 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:40:22,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:23,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:24,968 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:40:27,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:28,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:32,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:33,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:40:36,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:40:36,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:40:36,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:40:36,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:40:36,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:40:36,663 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:40:38,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:38,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:43,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:43,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:48,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:48,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:40:52,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:40:52,245 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:40:52,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:40:52,247 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:40:52,675 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:40:52,676 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:40:53,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:40:53,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:54,971 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:40:58,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:40:58,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:03,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:41:03,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:08,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:41:08,480 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:41:08,480 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:41:08,481 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:41:08,688 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:41:08,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:09,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:10,690 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:41:13,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:14,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:18,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:41:19,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:23,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:24,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:24,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:41:24,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:41:24,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:41:24,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:41:24,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:41:24,974 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:41:26,702 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:41:28,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:29,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:33,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:41:34,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:38,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:39,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:40,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:41:40,895 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:41:40,895 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:41:40,897 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:41:41,715 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:41:42,716 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:41:43,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:45,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:48,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:41:50,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:53,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:41:54,977 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:41:55,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:41:57,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:41:57,203 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:41:57,203 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:41:57,205 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:41:57,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:41:58,728 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:41:58,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:01,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:03,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:42:06,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:08,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:11,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:13,398 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:42:13,401 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:42:13,401 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:42:13,403 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:42:13,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:42:13,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:14,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:42:16,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:18,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:42:21,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:23,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:24,982 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:42:26,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:28,834 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:29,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:42:29,528 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:42:29,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:42:29,530 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:42:29,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:42:30,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:42:32,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:33,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:42:37,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:38,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:42,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:43,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:45,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:42:45,857 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:42:45,858 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:42:45,859 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:42:46,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:42:46,769 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:42:47,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:48,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:42:52,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:53,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:42:54,983 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:42:57,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:42:58,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:01,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:43:01,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:43:01,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:43:01,813 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:43:02,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:43:02,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:43:03,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:43:03,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:08,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:08,840 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:13,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:13,841 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:17,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:43:17,698 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:43:17,698 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:43:17,699 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:43:17,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:43:18,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:43:18,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:43:19,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:23,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:24,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:24,985 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:43:28,844 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:29,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:33,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:43:33,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:43:33,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:43:33,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:43:33,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:43:33,835 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:43:34,836 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:43:35,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:38,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:40,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:43,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:45,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:48,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:43:49,572 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:43:49,574 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:43:49,574 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:43:49,576 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:43:49,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:43:50,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:43:51,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:53,847 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:43:54,990 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:43:56,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:43:58,848 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:01,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:03,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:44:05,417 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:44:05,419 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:44:05,419 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:44:05,421 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:44:05,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:44:06,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:44:07,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:08,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:12,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:13,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:17,468 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:18,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:44:21,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:44:21,357 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:44:21,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:44:21,359 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:44:21,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:44:22,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:44:23,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:23,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:24,992 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:44:28,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:28,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:33,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:44:33,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:37,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:44:37,213 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:44:37,214 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:44:37,215 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:44:37,882 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:44:38,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:38,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:44:39,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:43,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:44,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:48,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:44:49,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:53,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:44:53,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:44:53,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:44:53,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:44:53,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:53,893 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:44:54,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:44:54,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:44:54,995 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:44:58,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:44:59,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:03,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:45:04,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:08,857 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:09,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:45:09,341 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:45:09,341 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:45:09,343 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:45:09,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:45:10,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:10,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:45:13,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:15,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:18,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:45:20,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:23,860 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:24,998 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:45:25,189 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:45:25,191 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:45:25,191 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:45:25,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:45:25,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:45:26,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:26,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:45:28,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:31,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:33,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:45:36,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:38,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:41,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:45:41,136 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:45:41,136 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:45:41,137 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:45:41,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:45:42,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:42,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:45:43,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:47,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:48,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:45:52,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:53,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:55,003 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:45:57,387 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:45:57,389 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:45:57,390 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:45:57,390 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:45:57,391 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:45:57,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:45:58,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:45:59,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:46:02,392 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:03,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:46:07,393 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:08,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:12,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:13,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:46:13,241 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:46:13,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:46:13,243 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:46:13,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:13,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:46:15,953 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:46:18,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:18,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:46:23,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:23,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:25,004 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:46:28,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:29,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:29,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:46:29,104 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:46:29,104 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:46:29,106 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:46:29,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:46:31,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:46:33,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:46:34,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:38,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:39,107 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:43,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:44,108 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:45,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:46:45,169 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:46:45,170 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:46:45,171 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:46:45,974 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:46:47,976 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:46:48,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:46:49,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:53,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:46:54,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:46:55,008 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:46:58,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:00,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:01,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:47:01,196 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:47:01,196 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:47:01,198 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:47:01,985 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:47:03,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:47:03,987 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:47:05,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:08,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:10,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:13,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:15,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:17,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:47:17,247 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:47:17,247 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:47:17,249 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:47:17,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:47:18,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:47:19,999 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:47:20,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:23,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:25,012 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:47:26,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:28,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:31,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:33,298 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:47:33,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:47:33,301 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:47:33,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:47:33,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:47:34,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:47:36,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:47:36,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:38,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:41,304 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:43,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:46,305 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:48,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:47:49,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:47:49,467 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:47:49,468 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:47:49,469 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:47:50,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:47:51,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:52,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:47:53,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:47:55,014 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:47:57,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:47:58,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:02,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:03,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:48:05,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:48:05,312 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:48:05,313 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:48:05,314 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:48:06,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:48:07,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:08,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:48:08,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:12,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:13,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:17,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:18,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:48:21,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:48:21,453 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:48:21,454 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:48:21,456 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:48:22,045 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:48:22,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:23,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:24,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:48:25,017 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:48:28,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:28,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:33,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:33,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:48:37,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:48:37,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:48:37,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:48:37,597 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:48:38,057 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:48:38,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:38,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:40,058 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:48:43,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:43,889 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:48,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:48,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:48:53,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:48:53,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:48:53,722 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:48:53,722 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:48:53,724 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:48:53,903 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:54,068 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:48:55,019 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:48:56,070 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:48:58,904 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:48:59,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:03,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:49:04,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:08,905 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:09,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:09,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:49:09,564 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:49:09,564 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:49:09,566 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:49:10,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:49:12,081 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:49:13,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:14,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:18,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:49:19,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:23,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:24,568 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:25,022 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:49:26,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:49:26,005 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:49:26,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:49:26,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:49:26,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:49:28,093 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:49:28,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:30,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:33,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:49:35,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:38,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:40,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:41,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:49:41,961 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:49:41,961 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:49:41,963 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:49:42,102 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:49:43,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:44,104 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:49:45,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:48,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:49:50,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:53,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:49:55,026 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:49:56,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:49:58,094 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:49:58,097 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:49:58,097 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:49:58,099 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:49:58,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:49:58,912 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:00,117 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:50:01,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:03,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:50:06,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:08,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:11,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:13,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:14,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:50:14,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:50:14,410 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:50:14,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:50:15,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:50:16,130 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:50:16,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:18,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:50:21,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:23,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:25,029 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:50:27,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:28,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:30,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:50:30,581 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:50:30,581 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:50:30,583 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:50:31,142 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:50:32,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:50:32,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:33,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:50:37,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:38,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:42,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:43,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:46,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:50:46,818 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:50:46,819 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:50:46,820 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:50:47,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:50:47,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:48,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:50:48,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:50:52,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:53,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:50:55,034 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:50:58,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:50:58,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:03,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:03,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:51:03,050 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:51:03,051 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:51:03,053 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:51:03,168 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:51:03,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:51:06,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:51:08,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:08,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:13,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:13,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:18,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:18,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:51:19,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:51:19,301 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:51:19,302 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:51:19,303 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:51:20,183 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:51:22,184 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:51:23,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:23,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:25,036 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:51:28,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:29,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:33,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:51:34,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:35,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:51:35,255 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:51:35,255 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:51:35,257 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:51:36,195 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:51:38,197 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:51:38,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:39,302 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:43,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:44,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:48,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:51:49,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:51,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:51:51,465 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:51:51,466 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:51:51,467 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:51:52,208 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:51:53,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:51:54,212 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:51:54,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:51:55,039 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:51:58,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:00,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:03,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:52:05,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:07,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:52:07,588 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:52:07,588 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:52:07,590 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:52:08,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:52:08,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:10,225 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:52:10,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:13,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:15,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:18,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:52:20,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:23,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:52:23,655 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:52:23,655 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:52:23,657 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:52:23,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:24,236 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:52:25,043 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:52:26,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:26,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:52:28,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:31,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:33,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:52:36,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:38,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:39,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:52:39,811 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:52:39,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:52:39,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:52:40,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:52:41,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:42,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:52:43,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:46,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:48,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:52:51,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:53,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:52:55,045 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:52:55,933 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:52:55,936 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:52:55,937 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:52:55,938 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:52:56,262 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:52:56,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:52:58,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:52:58,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:01,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:03,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:53:06,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:08,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:11,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:12,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:53:12,084 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:53:12,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:53:12,086 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:53:12,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:53:13,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:14,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:53:17,087 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:18,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:53:22,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:23,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:25,048 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:53:28,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:28,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:53:28,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:53:28,133 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:53:28,134 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:53:28,287 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:53:28,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:30,288 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:53:33,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:33,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:53:38,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:38,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:43,136 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:43,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:44,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:53:44,243 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:53:44,243 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:53:44,245 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:53:44,301 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:53:46,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:53:48,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:48,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:53:53,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:53:53,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:55,051 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:53:58,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:53:59,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:00,105 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:54:00,108 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:54:00,109 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:54:00,110 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:54:00,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:54:02,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:54:03,737 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:54:04,111 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:08,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:09,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:13,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:14,112 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:15,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:54:15,969 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:54:15,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:54:15,971 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:54:16,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:54:18,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:54:18,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:54:19,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:23,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:24,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:25,054 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:54:28,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:30,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:32,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:54:32,113 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:54:32,113 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:54:32,115 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:54:32,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:54:33,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:54:34,344 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:54:35,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:38,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:40,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:43,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:45,117 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:47,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:54:47,967 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:54:47,967 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:54:47,969 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:54:48,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:54:48,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:54:50,361 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:54:50,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:53,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:54:55,056 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:54:56,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:54:58,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:01,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:03,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:55:03,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:55:03,816 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:55:03,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:55:03,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:55:04,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:55:06,374 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:55:06,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:08,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:11,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:13,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:16,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:18,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:55:19,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:55:19,957 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:55:19,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:55:19,959 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:55:20,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:55:21,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:22,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:55:23,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:25,060 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:55:27,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:28,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:32,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:33,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:55:35,810 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:55:35,812 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:55:35,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:55:35,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:55:36,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:55:37,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:38,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:55:38,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:42,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:43,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:47,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:48,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:55:51,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:55:51,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:55:51,762 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:55:51,764 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:55:52,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:55:53,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:55:53,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:54,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:55:55,064 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:55:58,969 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:55:59,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:03,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:56:04,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:08,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:56:08,132 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:56:08,132 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:56:08,134 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:56:08,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:56:08,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:09,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:10,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:56:13,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:14,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:18,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:56:19,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:23,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:24,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:56:24,016 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:56:24,016 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:56:24,017 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:56:24,439 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:56:25,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:25,066 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:56:26,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:56:28,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:30,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:33,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:56:35,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:38,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:39,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:56:39,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:56:39,833 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:56:39,835 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:56:40,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:56:40,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:42,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:56:43,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:45,836 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:48,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:56:50,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:53,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:56:55,069 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:56:55,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:56:55,983 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:56:55,983 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:56:55,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:56:55,985 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:56:56,465 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:56:58,466 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:56:58,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:00,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:03,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:57:05,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:08,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:10,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:11,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:57:11,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:57:11,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:57:11,929 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:57:12,478 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:57:13,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:14,479 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:57:16,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:18,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:57:21,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:23,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:25,073 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:57:27,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:28,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:57:28,214 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:57:28,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:57:28,216 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:57:28,494 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:57:28,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:30,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:57:32,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:33,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:57:37,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:38,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:42,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:43,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:44,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:57:44,194 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:57:44,195 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:57:44,196 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:57:44,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:57:46,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:57:48,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:48,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:57:53,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:57:53,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:55,076 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:57:58,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:57:59,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:00,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:58:00,445 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:58:00,446 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:58:00,447 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:58:00,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:58:02,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:58:03,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:58:04,448 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:08,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:09,449 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:13,989 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:14,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:16,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:58:16,309 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:58:16,309 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:58:16,311 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:58:16,536 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:58:18,537 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:58:18,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:58:20,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:23,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:25,079 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:58:26,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:28,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:31,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:32,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:58:32,485 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:58:32,485 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:58:32,487 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:58:32,550 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:58:33,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:58:34,551 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:58:36,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:38,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:41,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:43,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:46,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:48,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:58:48,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:58:48,874 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:58:48,875 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:58:48,876 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:58:49,564 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:58:50,565 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:58:51,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:53,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:58:55,082 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:58:57,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:58:58,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:02,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:03,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:59:04,842 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:59:04,844 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:59:04,844 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:59:04,846 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:59:05,576 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:59:06,577 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:59:07,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:08,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:12,887 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:13,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:17,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:18,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:59:20,881 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:59:20,883 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:59:20,883 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:59:20,885 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:59:21,589 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:59:22,590 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:59:22,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:24,000 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:25,112 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:59:28,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:29,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:33,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:33,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:59:37,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:59:37,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:59:37,270 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:59:37,273 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:59:37,601 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:59:38,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:38,602 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:59:39,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:43,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:44,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:48,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:48,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 15:59:53,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 15:59:53,111 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 15:59:53,112 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 15:59:53,113 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 15:59:53,619 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 15:59:54,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 15:59:54,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 15:59:54,620 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 15:59:55,088 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 15:59:59,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:00,091 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:03,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:00:05,092 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:09,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:09,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:00:09,155 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:00:09,156 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:00:09,158 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:00:09,632 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:00:10,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:10,633 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:00:14,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:15,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:18,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:00:20,160 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:24,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:25,090 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:00:25,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:00:25,321 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:00:25,321 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:00:25,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:25,323 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:00:25,644 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:00:26,645 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:00:29,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:30,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:33,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:00:35,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:39,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:40,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:41,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:00:41,275 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:00:41,276 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:00:41,278 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:00:41,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:00:42,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:00:44,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:46,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:48,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:00:51,280 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:54,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:00:55,094 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:00:57,095 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:00:57,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:00:57,439 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:00:57,439 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:00:57,441 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:00:57,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:00:58,669 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:00:59,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:02,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:03,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:01:07,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:09,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:12,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:13,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:01:13,474 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:01:13,475 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:01:13,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:01:13,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:01:14,016 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:14,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:01:17,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:18,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:01:22,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:24,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:25,131 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:01:28,133 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:29,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:29,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:01:29,604 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:01:29,604 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:01:29,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:01:29,693 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:01:30,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:01:33,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:33,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:01:38,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:39,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:43,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:44,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:45,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:01:45,833 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:01:45,834 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:01:45,836 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:01:46,709 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:01:48,711 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:01:48,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:01:48,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:53,837 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:01:54,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:55,134 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:01:59,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:01:59,135 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:01,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:02:01,960 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:02:01,960 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:02:01,962 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:02:02,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:02:03,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:02:04,724 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:02:04,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:09,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:09,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:14,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:14,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:18,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:02:18,232 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:02:18,233 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:02:18,235 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:02:18,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:02:18,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:02:20,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:20,737 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:02:24,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:25,137 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:02:26,138 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:29,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:31,139 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:33,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:02:34,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:02:34,188 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:02:34,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:02:34,190 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:02:34,748 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:02:36,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:36,750 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:02:39,030 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:41,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:44,032 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:46,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:48,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:02:50,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:02:50,347 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:02:50,347 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:02:50,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:02:50,762 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:02:51,391 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:52,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:02:54,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:02:55,141 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:02:57,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:02:59,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:02,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:03,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:03:06,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:03:06,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:03:06,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:03:06,667 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:03:06,774 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:03:07,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:08,775 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:03:09,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:12,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:14,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:17,711 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:18,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:03:22,712 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:22,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:03:22,730 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:03:22,730 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:03:22,733 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:03:22,786 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:03:24,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:24,788 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:03:25,143 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:03:28,144 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:29,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:33,145 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:33,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:03:38,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:38,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:03:38,670 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:03:38,671 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:03:38,672 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:03:38,799 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:03:39,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:40,800 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:03:43,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:44,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:48,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:48,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:03:53,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:03:54,043 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:54,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:03:54,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:03:54,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:03:54,973 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:03:55,146 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:03:55,813 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:03:56,813 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:03:59,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:03:59,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:03,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:04:04,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:09,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:09,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:10,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:04:10,805 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:04:10,805 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:04:10,807 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:04:10,824 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:04:12,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:04:14,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:14,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:18,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:04:19,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:24,048 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:24,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:25,150 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:04:26,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:04:26,765 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:04:26,765 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:04:26,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:04:26,837 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:04:28,839 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:04:29,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:30,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:33,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:04:35,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:39,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:40,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:43,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:04:43,005 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:04:43,005 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:04:43,007 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:04:43,852 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:04:44,051 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:44,853 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:04:46,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:48,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:04:51,009 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:54,052 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:04:55,152 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:04:56,153 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:04:58,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:04:58,852 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:04:58,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:04:58,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:04:58,865 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:04:59,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:00,866 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:05:01,854 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:03,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:05:06,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:09,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:11,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:14,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:14,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:05:14,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:05:14,931 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:05:14,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:05:15,879 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:05:16,880 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:05:16,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:18,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:05:21,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:24,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:25,155 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:05:27,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:29,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:31,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:05:31,008 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:05:31,008 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:05:31,010 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:05:31,891 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:05:32,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:05:33,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:33,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:05:38,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:39,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:43,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:44,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:46,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:05:46,856 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:05:46,856 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:05:46,857 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:05:46,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:05:48,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:05:48,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:48,906 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:05:53,859 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:05:54,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:55,157 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:05:59,062 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:05:59,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:02,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:06:02,974 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:06:02,975 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:06:02,976 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:06:03,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:06:03,916 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:06:04,917 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:06:04,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:09,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:09,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:14,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:14,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:18,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:06:19,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:06:19,026 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:06:19,026 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:06:19,027 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:06:19,929 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:06:20,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:20,930 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:06:24,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:25,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:25,161 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:06:29,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:30,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:33,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:06:35,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:35,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:06:35,172 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:06:35,172 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:06:35,173 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:06:35,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:06:36,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:06:39,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:40,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:44,069 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:45,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:48,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:06:50,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:51,121 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:06:51,123 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:06:51,123 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:06:51,125 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:06:51,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:06:52,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:06:54,070 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:06:55,164 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:06:56,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:06:59,071 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:01,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:03,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:07:06,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:07,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:07:07,318 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:07:07,318 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:07:07,320 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:07:07,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:07:08,968 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:07:09,072 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:11,363 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:14,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:16,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:18,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:07:21,364 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:23,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:07:23,516 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:07:23,517 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:07:23,518 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:07:23,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:07:24,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:24,981 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:07:25,167 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:07:27,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:29,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:32,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:33,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:07:37,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:39,077 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:39,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:07:39,596 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:07:39,597 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:07:39,598 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:07:39,995 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:07:40,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:07:42,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:44,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:47,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:48,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:07:52,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:54,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:07:55,172 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:07:55,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:07:55,629 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:07:55,629 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:07:55,630 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:07:56,008 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:07:57,009 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:07:57,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:07:59,080 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:02,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:03,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:08:07,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:09,081 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:11,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:08:11,951 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:08:11,951 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:08:11,953 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:08:12,021 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:08:12,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:13,022 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:08:14,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:17,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:18,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:08:22,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:24,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:25,173 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:08:27,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:08:27,953 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:08:27,953 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:08:27,954 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:08:27,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:28,034 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:08:29,035 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:08:29,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:32,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:33,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:08:37,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:39,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:42,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:43,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:08:43,912 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:08:43,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:08:43,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:08:44,046 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:08:44,086 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:45,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:08:48,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:08:48,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:53,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:08:54,088 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:55,177 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:08:59,089 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:08:59,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:00,148 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:09:00,151 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:09:00,151 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:09:00,153 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:09:01,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:09:01,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:09:03,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:09:05,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:09,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:10,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:14,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:15,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:16,456 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:09:16,459 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:09:16,459 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:09:16,461 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:09:17,074 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:09:18,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:09:19,075 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:09:20,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:24,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:25,181 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:09:26,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:29,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:31,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:32,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:09:32,404 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:09:32,404 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:09:32,406 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:09:33,086 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:09:33,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:09:35,087 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:09:36,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:39,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:41,407 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:44,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:46,408 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:48,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:09:48,638 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:09:48,638 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:09:48,640 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:09:48,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:09:49,099 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:09:51,100 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:09:51,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:54,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:09:55,183 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:09:57,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:09:59,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:02,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:03,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:10:04,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:10:04,589 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:10:04,589 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:10:04,591 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:10:05,112 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:10:07,113 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:10:07,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:09,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:12,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:14,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:17,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:18,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:10:20,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:10:20,645 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:10:20,646 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:10:20,647 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:10:21,126 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:10:22,690 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:23,127 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:10:24,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:25,186 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:10:28,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:29,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:33,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:33,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:10:37,093 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:10:37,095 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:10:37,096 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:10:37,097 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:10:37,139 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:10:39,140 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:10:39,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:39,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:44,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:44,472 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:48,744 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:10:49,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:53,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:10:53,278 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:10:53,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:10:53,280 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:10:54,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:10:54,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:10:54,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:10:55,154 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:10:55,189 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:10:59,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:00,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:03,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:11:05,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:09,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:11:09,332 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:11:09,333 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:11:09,334 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:11:09,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:10,166 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:11:10,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:11,167 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:11:14,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:15,375 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:18,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:11:20,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:24,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:25,193 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:11:25,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:11:25,458 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:11:25,458 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:11:25,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:25,460 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:11:26,179 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:11:27,180 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:11:29,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:30,461 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:33,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:11:35,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:39,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:40,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:41,792 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:11:41,795 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:11:41,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:11:41,797 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:11:42,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:11:43,193 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:11:44,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:45,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:48,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:11:50,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:54,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:11:55,194 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:11:56,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:11:57,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:11:57,958 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:11:57,959 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:11:57,960 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:11:58,210 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:11:59,211 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:11:59,483 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:01,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:03,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:12:06,962 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:09,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:11,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:14,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:12:14,197 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:12:14,197 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:12:14,199 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:12:14,223 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:12:14,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:15,224 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:12:17,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:18,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:12:22,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:24,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:25,198 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:12:28,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:29,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:30,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:12:30,296 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:12:30,297 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:12:30,298 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:12:31,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:12:31,238 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:12:33,299 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:33,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:12:38,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:39,489 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:43,300 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:44,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:46,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:12:46,354 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:12:46,354 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:12:46,356 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:12:47,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:12:47,250 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:12:48,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:48,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:12:53,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:54,491 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:12:55,202 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:12:59,203 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:12:59,492 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:02,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:13:02,398 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:13:02,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:13:02,399 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:13:03,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:13:03,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:13:03,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:13:04,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:09,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:09,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:14,401 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:14,494 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:18,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:13:18,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:13:18,817 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:13:18,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:13:18,818 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:13:19,278 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:13:19,862 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:21,279 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:13:24,496 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:24,863 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:25,204 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:13:29,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:30,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:33,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:13:34,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:13:34,837 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:13:34,837 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:13:34,839 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:13:35,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:13:35,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:37,292 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:13:39,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:40,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:44,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:45,879 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:48,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:13:50,880 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:50,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:13:50,887 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:13:50,887 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:13:50,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:13:51,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:13:53,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:13:54,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:13:55,207 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:13:56,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:13:59,502 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:01,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:03,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:14:06,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:07,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:14:07,218 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:14:07,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:14:07,220 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:14:07,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:14:09,319 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:14:09,503 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:11,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:14,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:16,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:18,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:14:21,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:23,061 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:14:23,063 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:14:23,063 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:14:23,065 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:14:23,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:14:24,505 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:25,210 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:14:25,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:14:27,211 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:29,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:32,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:33,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:14:37,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:38,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:14:38,947 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:14:38,948 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:14:38,949 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:14:39,343 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:14:39,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:41,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:14:42,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:44,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:47,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:48,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:14:52,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:54,509 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:14:55,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:14:55,212 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:14:55,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:14:55,216 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:14:55,216 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:14:55,357 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:14:57,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:14:58,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:14:59,510 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:03,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:03,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:15:08,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:09,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:11,068 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:15:11,071 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:15:11,071 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:15:11,073 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:15:11,371 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:15:13,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:15:14,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:14,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:18,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:15:19,074 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:24,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:24,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:25,217 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:15:27,120 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:15:27,123 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:15:27,123 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:15:27,124 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:15:27,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:15:29,125 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:29,386 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:15:29,515 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:33,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:15:34,126 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:39,127 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:39,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:43,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:15:43,079 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:15:43,080 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:15:43,081 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:15:43,397 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:15:44,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:45,082 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:45,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:15:48,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:15:50,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:54,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:15:55,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:15:55,222 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:15:59,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:15:59,020 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:15:59,021 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:15:59,022 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:15:59,411 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:15:59,519 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:01,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:01,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:16:03,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:16:06,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:09,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:11,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:14,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:15,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:16:15,160 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:16:15,161 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:16:15,162 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:16:15,423 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:16:16,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:17,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:16:18,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:16:21,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:24,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:25,226 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:16:26,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:29,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:31,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:31,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:16:31,339 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:16:31,340 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:16:31,341 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:16:31,436 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:16:33,437 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:16:33,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:16:36,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:39,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:41,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:44,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:46,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:47,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:16:47,586 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:16:47,586 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:16:47,588 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:16:48,450 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:16:48,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:16:49,451 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:16:51,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:54,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:16:55,228 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:16:57,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:16:59,529 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:02,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:03,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:17:03,534 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:17:03,534 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:17:03,536 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:17:03,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:17:04,463 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:17:05,464 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:17:07,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:09,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:12,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:14,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:17,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:18,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:17:19,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:17:19,590 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:17:19,590 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:17:19,592 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:17:20,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:17:21,477 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:17:22,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:24,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:25,230 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:17:28,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:29,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:33,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:33,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:17:35,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:17:35,804 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:17:35,804 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:17:35,806 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:17:36,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:17:37,490 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:17:38,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:39,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:43,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:44,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:48,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:17:48,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:51,871 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:17:51,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:17:51,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:17:51,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:17:52,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:17:53,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:17:53,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:54,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:17:55,233 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:17:59,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:17:59,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:03,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:18:04,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:08,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:18:08,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:18:08,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:18:08,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:18:08,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:18:09,515 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:18:09,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:10,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:14,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:15,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:18,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:18:20,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:24,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:18:24,381 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:18:24,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:18:24,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:18:24,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:18:24,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:25,271 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:18:25,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:25,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:18:29,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:30,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:33,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:18:35,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:39,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:40,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:40,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:18:40,317 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:18:40,317 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:18:40,319 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:18:40,541 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:18:41,542 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:18:44,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:45,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:48,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:18:50,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:54,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:18:55,274 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:18:56,275 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:18:56,353 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:18:56,356 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:18:56,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:18:56,358 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:18:56,554 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:18:57,555 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:18:59,546 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:01,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:03,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:19:06,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:09,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:11,361 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:12,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:19:12,545 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:19:12,546 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:19:12,547 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:19:12,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:19:13,568 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:19:14,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:16,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:18,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:19:21,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:24,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:25,277 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:19:27,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:28,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:19:28,606 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:19:28,607 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:19:28,609 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:19:29,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:29,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:19:29,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:19:32,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:33,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:19:37,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:39,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:42,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:44,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:44,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:19:44,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:19:44,579 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:19:44,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:19:44,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:19:45,596 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:19:48,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:48,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:19:53,582 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:54,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:19:55,281 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:19:59,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:19:59,556 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:00,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:20:00,801 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:20:00,801 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:20:00,803 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:20:01,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:20:01,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:20:03,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:20:04,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:09,557 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:09,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:14,558 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:14,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:16,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:20:16,835 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:20:16,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:20:16,837 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:20:17,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:20:17,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:20:18,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:20:19,838 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:24,560 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:24,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:25,283 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:20:29,561 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:30,285 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:32,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:20:32,991 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:20:32,992 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:20:32,993 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:20:33,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:20:33,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:20:33,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:20:35,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:39,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:40,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:44,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:45,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:48,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:20:49,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:20:49,335 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:20:49,335 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:20:49,337 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:20:49,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:20:51,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:51,653 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:20:54,564 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:20:55,287 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:20:57,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:20:59,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:02,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:03,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:21:05,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:21:05,382 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:21:05,382 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:21:05,383 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:21:05,665 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:21:07,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:07,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:21:09,566 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:12,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:14,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:17,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:18,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:21:21,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:21:21,543 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:21:21,543 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:21:21,544 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:21:21,678 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:21:22,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:23,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:21:24,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:25,291 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:21:28,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:29,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:33,293 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:33,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:21:37,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:21:37,688 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:21:37,689 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:21:37,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:21:37,692 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:21:38,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:39,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:21:39,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:43,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:44,727 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:48,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:48,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:21:53,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:53,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:21:53,895 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:21:53,895 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:21:53,897 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:21:54,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:21:54,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:21:55,293 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:21:55,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:21:59,294 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:21:59,729 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:03,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:22:04,295 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:09,296 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:09,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:10,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:22:10,052 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:22:10,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:22:10,054 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:22:10,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:22:11,723 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:22:14,732 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:15,055 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:18,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:22:20,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:24,733 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:25,057 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:25,298 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:22:26,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:22:26,212 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:22:26,213 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:22:26,214 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:22:26,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:22:27,736 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:22:29,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:30,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:33,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:22:35,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:39,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:40,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:42,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:22:42,461 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:22:42,462 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:22:42,464 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:22:42,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:22:43,749 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:22:44,736 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:45,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:48,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:22:50,465 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:54,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:55,300 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:22:56,301 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:22:58,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:22:58,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:22:58,409 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:22:58,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:22:58,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:22:59,739 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:22:59,763 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:23:01,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:03,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:23:06,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:09,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:11,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:14,453 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:23:14,455 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:23:14,456 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:23:14,457 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:23:14,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:14,777 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:23:15,778 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:23:16,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:18,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:23:21,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:24,742 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:25,304 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:23:27,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:29,743 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:30,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:23:30,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:23:30,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:23:30,668 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:23:30,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:23:31,791 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:23:32,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:33,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:23:37,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:39,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:42,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:44,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:46,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:23:46,712 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:23:46,712 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:23:46,714 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:23:46,804 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:23:47,715 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:47,805 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:23:48,749 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:23:52,716 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:54,747 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:23:55,308 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:23:58,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:23:59,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:02,861 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:24:02,864 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:24:02,865 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:24:02,866 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:24:03,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:24:03,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:24:03,820 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:24:03,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:08,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:09,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:13,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:14,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:18,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:24:18,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:19,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:24:19,186 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:24:19,186 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:24:19,188 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:24:19,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:24:19,833 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:24:23,752 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:24,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:25,311 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:24:28,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:29,312 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:33,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:24:34,313 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:35,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:24:35,028 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:24:35,028 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:24:35,030 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:24:35,846 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:24:35,847 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:24:38,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:40,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:43,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:45,075 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:48,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:24:50,076 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:51,079 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:24:51,080 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:24:51,081 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:24:51,082 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:24:51,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:24:51,861 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:24:53,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:55,315 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:24:55,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:24:55,864 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:24:59,149 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:24:59,867 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:01,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:01,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:03,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:25:06,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:07,874 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:09,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:09,876 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:11,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:25:11,878 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:12,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:14,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:17,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:18,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:25:22,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:24,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:25,318 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:25:27,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:27,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:25:27,466 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:25:27,467 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:25:27,468 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:25:27,892 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:25:29,581 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:29,894 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:32,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:33,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:25:37,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:39,583 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:42,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:43,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:25:43,919 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:25:43,920 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:25:43,921 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:25:44,584 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:44,907 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:25:45,908 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:25:47,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:48,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:25:52,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:54,585 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:55,321 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:25:58,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:25:59,586 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:25:59,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:25:59,870 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:25:59,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:25:59,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:25:59,920 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:26:01,922 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:26:03,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:26:03,873 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:08,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:09,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:13,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:14,588 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:16,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:26:16,191 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:26:16,192 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:26:16,193 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:26:16,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:26:17,936 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:26:18,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:26:19,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:24,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:24,589 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:25,325 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:26:29,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:29,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:32,406 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:26:32,409 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:26:32,409 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:26:32,411 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:26:32,950 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:26:33,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:26:33,951 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:26:34,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:39,412 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:39,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:44,413 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:44,593 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:48,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:26:48,270 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:26:48,271 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:26:48,272 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:26:48,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:26:48,964 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:26:49,965 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:26:50,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:54,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:26:55,274 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:26:55,327 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:26:59,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:00,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:03,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:27:04,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:27:04,508 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:27:04,509 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:27:04,510 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:27:04,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:27:05,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:05,978 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:27:09,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:10,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:14,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:15,555 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:18,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:27:20,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:27:20,550 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:27:20,550 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:27:20,552 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:27:20,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:20,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:27:21,992 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:27:24,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:25,330 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:27:26,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:29,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:31,332 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:33,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:27:36,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:36,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:27:36,613 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:27:36,613 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:27:36,614 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:27:37,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:27:38,006 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:27:39,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:41,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:44,602 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:46,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:48,776 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:27:51,660 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:52,565 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:27:52,567 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:27:52,567 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:27:52,569 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:27:53,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:27:54,020 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:27:54,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:27:55,333 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:27:57,334 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:27:59,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:02,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:03,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:28:07,336 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:08,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:28:08,506 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:28:08,506 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:28:08,508 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:28:09,032 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:28:09,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:10,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:28:12,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:14,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:17,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:18,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:28:22,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:24,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:24,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:28:24,666 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:28:24,666 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:28:24,668 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:28:25,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:28:25,335 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:28:26,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:28:28,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:29,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:33,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:33,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:28:38,338 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:39,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:40,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:28:40,737 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:28:40,737 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:28:40,739 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:28:41,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:28:42,062 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:28:43,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:44,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:48,740 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:48,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:28:53,741 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:54,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:28:55,339 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:28:56,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:28:56,917 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:28:56,917 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:28:56,919 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:28:57,076 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:28:58,077 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:28:58,919 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:28:59,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:03,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:29:03,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:08,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:09,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:12,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:29:12,764 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:29:12,765 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:29:12,767 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:29:13,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:29:14,091 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:29:14,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:14,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:18,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:29:19,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:24,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:24,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:25,342 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:29:28,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:29:28,717 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:29:28,717 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:29:28,719 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:29:29,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:29:29,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:30,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:29:30,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:33,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:29:35,720 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:39,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:40,721 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:44,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:44,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:29:44,956 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:29:44,956 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:29:44,958 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:29:45,119 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:29:45,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:46,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:29:48,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:29:50,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:54,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:29:55,346 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:29:56,347 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:29:59,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:01,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:30:01,164 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:30:01,164 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:30:01,166 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:30:02,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:30:02,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:30:02,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:03,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:30:07,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:09,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:12,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:14,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:17,116 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:30:17,119 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:30:17,119 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:30:17,121 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:30:17,147 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:30:18,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:18,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:30:18,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:30:23,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:24,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:25,348 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:30:28,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:29,629 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:33,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:30:33,155 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:30:33,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:30:33,157 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:30:33,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:30:33,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:30:34,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:34,162 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:30:39,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:39,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:44,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:44,632 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:48,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:30:49,100 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:30:49,102 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:30:49,102 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:30:49,104 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:30:49,174 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:30:50,142 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:50,175 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:30:54,633 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:30:55,143 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:30:55,351 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:30:59,634 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:00,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:03,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:31:05,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:31:05,338 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:31:05,338 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:31:05,340 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:31:05,382 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:06,189 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:31:06,190 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:31:09,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:10,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:14,636 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:15,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:18,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:31:20,384 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:21,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:31:21,490 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:31:21,490 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:31:21,492 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:31:22,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:31:22,204 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:31:24,638 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:25,355 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:31:26,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:29,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:31,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:33,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:31:36,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:37,604 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:31:37,608 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:31:37,608 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:31:37,610 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:31:38,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:31:38,217 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:31:39,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:41,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:44,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:46,651 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:48,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:31:51,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:53,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:31:53,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:31:53,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:31:53,474 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:31:54,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:31:54,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:31:54,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:31:55,357 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:31:57,358 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:31:59,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:02,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:03,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:32:07,360 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:09,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:32:09,624 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:32:09,624 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:32:09,626 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:32:09,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:10,244 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:32:10,245 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:32:12,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:14,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:17,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:18,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:32:22,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:24,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:25,361 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:32:25,755 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:32:25,758 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:32:25,758 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:32:25,759 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:32:26,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:32:26,259 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:32:27,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:29,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:32,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:33,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:32:37,761 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:39,649 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:41,703 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:32:41,705 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:32:41,706 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:32:41,708 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:32:42,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:32:42,273 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:32:43,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:44,650 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:48,709 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:48,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:32:53,710 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:54,652 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:32:55,364 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:32:57,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:32:57,970 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:32:57,970 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:32:57,972 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:32:58,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:32:58,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:32:59,653 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:00,291 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:33:03,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:33:03,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:08,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:09,654 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:13,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:14,239 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:33:14,242 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:33:14,242 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:33:14,244 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:33:14,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:33:14,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:16,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:33:18,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:33:19,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:24,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:24,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:25,368 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:33:29,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:29,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:30,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:33:30,463 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:33:30,463 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:33:30,465 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:33:31,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:33:32,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:33:33,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:33:34,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:39,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:39,658 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:44,467 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:44,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:46,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:33:46,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:33:46,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:33:46,322 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:33:46,330 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:33:48,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:33:48,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:33:50,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:54,661 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:33:55,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:33:55,370 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:33:59,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:00,371 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:02,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:34:02,769 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:34:02,770 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:34:02,771 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:34:03,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:34:03,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:34:04,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:34:05,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:09,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:10,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:14,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:15,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:18,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:34:18,733 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:34:18,733 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:34:18,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:34:18,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:34:19,362 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:34:20,363 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:34:21,735 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:24,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:25,372 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:34:27,373 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:29,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:32,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:33,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:34:34,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:34:34,886 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:34:34,887 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:34:34,888 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:34:35,378 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:34:36,379 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:34:37,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:39,668 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:42,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:44,669 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:47,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:48,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:34:50,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:34:50,920 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:34:50,920 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:34:50,922 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:34:51,391 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:34:52,392 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:34:52,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:54,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:34:55,376 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:34:58,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:34:59,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:03,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:03,779 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:35:07,140 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:35:07,142 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:35:07,142 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:35:07,144 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:35:07,405 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:35:08,406 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:35:09,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:09,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:14,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:14,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:18,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:35:19,187 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:23,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:35:23,274 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:35:23,274 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:35:23,275 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:35:23,419 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:35:24,314 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:24,420 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:35:24,675 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:25,378 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:35:29,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:29,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:33,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:35:34,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:39,381 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:39,405 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:35:39,407 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:35:39,407 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:35:39,408 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:35:39,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:35:39,677 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:40,433 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:35:44,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:44,678 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:48,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:35:49,451 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:54,452 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:35:54,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:55,418 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:35:55,708 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:35:55,710 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:35:55,711 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:35:55,712 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:35:56,447 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:35:56,448 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:35:59,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:35:59,713 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:03,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:36:04,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:09,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:09,714 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:11,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:36:11,872 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:36:11,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:36:11,874 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:36:12,460 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:36:12,461 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:36:14,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:14,875 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:18,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:36:19,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:24,684 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:24,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:25,421 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:36:28,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:36:28,174 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:36:28,175 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:36:28,176 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:36:28,474 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:36:29,685 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:30,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:30,476 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:36:33,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:36:35,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:39,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:40,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:44,283 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:36:44,285 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:36:44,286 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:36:44,288 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:36:44,488 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:36:44,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:45,288 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:46,489 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:36:48,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:36:50,289 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:54,688 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:36:55,290 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:36:55,426 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:36:59,689 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:00,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:37:00,238 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:37:00,239 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:37:00,240 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:37:00,502 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:37:01,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:02,503 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:37:03,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:37:06,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:09,691 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:11,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:14,692 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:16,243 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:16,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:37:16,444 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:37:16,445 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:37:16,446 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:37:16,516 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:37:18,517 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:37:18,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:37:21,447 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:24,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:25,428 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:37:27,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:29,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:32,429 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:32,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:37:32,571 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:37:32,572 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:37:32,573 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:37:33,530 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:37:33,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:37:34,531 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:37:37,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:39,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:42,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:44,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:47,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:48,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:37:48,432 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:37:48,433 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:37:48,434 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:37:48,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:37:48,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:37:50,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:37:53,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:54,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:37:55,431 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:37:59,432 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:37:59,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:03,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:38:04,374 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:38:04,376 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:38:04,376 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:38:04,378 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:38:04,556 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:38:05,422 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:06,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:38:08,794 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:10,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:13,795 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:15,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:18,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:38:20,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:38:20,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:38:20,420 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:38:20,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:38:20,466 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:20,570 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:38:22,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:38:23,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:25,435 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:38:26,436 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:28,797 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:31,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:33,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:38:36,437 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:36,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:38:36,582 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:38:36,583 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:38:36,584 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:38:37,585 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:38:38,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:38:38,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:41,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:43,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:46,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:48,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:38:51,628 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:52,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:38:52,542 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:38:52,542 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:38:52,544 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:38:52,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:38:53,801 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:38:54,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:38:55,437 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:38:57,438 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:38:58,802 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:02,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:03,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:39:07,439 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:08,804 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:08,845 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:39:08,848 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:39:08,848 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:39:08,849 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:39:09,611 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:39:10,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:39:12,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:13,805 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:17,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:18,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:39:22,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:23,806 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:24,659 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:39:24,662 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:39:24,662 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:39:24,664 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:39:25,440 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:39:25,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:39:26,627 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:39:28,441 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:28,807 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:33,442 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:33,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:39:38,443 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:38,808 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:40,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:39:40,921 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:39:40,922 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:39:40,923 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:39:41,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:39:42,639 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:39:43,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:43,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:48,781 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:39:48,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:53,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:53,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:39:55,442 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:39:56,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:39:56,895 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:39:56,895 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:39:56,897 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:39:57,654 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:39:58,655 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:39:58,811 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:39:59,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:03,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:40:04,898 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:08,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:09,899 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:13,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:40:13,232 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:40:13,232 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:40:13,234 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:40:13,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:40:13,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:14,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:40:15,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:18,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:40:20,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:23,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:25,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:25,446 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:40:28,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:29,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:40:29,188 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:40:29,188 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:40:29,190 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:40:29,681 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:40:30,682 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:40:31,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:33,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:40:36,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:38,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:41,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:43,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:45,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:40:45,225 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:40:45,226 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:40:45,227 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:40:45,698 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:40:46,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:46,699 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:40:48,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:40:51,228 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:53,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:40:55,449 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:40:56,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:40:58,821 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:01,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:41:01,257 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:41:01,258 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:41:01,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:41:01,713 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:41:02,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:02,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:41:03,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:41:07,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:08,822 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:12,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:13,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:17,262 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:17,378 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:41:17,380 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:41:17,381 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:41:17,382 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:41:17,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:41:18,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:41:18,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:41:22,383 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:23,824 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:25,453 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:41:27,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:28,825 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:32,455 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:33,414 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:41:33,417 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:41:33,418 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:41:33,419 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:41:33,740 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:41:33,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:41:34,741 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:41:38,420 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:38,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:43,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:43,827 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:48,421 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:48,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:41:49,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:41:49,545 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:41:49,545 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:41:49,546 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:41:49,754 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:41:50,755 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:41:53,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:41:53,829 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:55,457 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:41:58,830 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:41:59,458 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:03,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:42:04,459 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:05,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:42:05,482 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:42:05,483 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:42:05,484 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:42:05,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:42:06,768 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:42:08,831 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:09,526 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:13,832 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:14,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:18,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:42:19,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:21,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:42:21,518 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:42:21,519 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:42:21,520 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:42:21,781 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:42:22,782 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:42:23,833 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:24,562 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:25,458 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:42:28,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:30,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:33,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:42:35,460 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:37,839 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:42:37,841 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:42:37,841 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:42:37,843 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:42:38,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:42:38,797 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:42:38,908 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:40,882 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:43,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:45,883 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:48,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:42:50,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:53,681 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:42:53,683 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:42:53,683 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:42:53,685 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:42:53,811 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:42:53,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:42:54,812 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:42:55,461 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:42:56,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:42:58,911 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:01,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:03,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:43:06,464 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:08,913 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:09,846 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:43:09,849 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:43:09,849 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:43:09,851 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:43:10,825 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:43:10,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:43:11,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:13,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:16,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:18,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:43:21,853 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:23,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:25,465 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:43:25,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:43:25,890 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:43:25,890 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:43:25,892 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:43:26,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:43:26,840 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:43:26,892 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:28,916 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:31,893 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:33,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:43:36,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:38,917 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:41,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:42,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:43:42,062 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:43:42,062 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:43:42,063 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:43:42,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:43:42,854 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:43:43,918 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:47,064 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:48,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:43:52,065 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:53,920 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:43:55,468 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:43:57,469 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:43:58,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:43:58,015 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:43:58,016 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:43:58,017 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:43:58,869 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:43:58,870 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:43:58,921 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:03,018 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:03,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:44:08,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:08,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:13,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:13,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:14,370 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:44:14,373 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:44:14,373 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:44:14,375 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:44:14,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:44:14,883 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:44:18,376 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:18,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:44:23,377 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:23,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:25,472 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:44:28,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:28,926 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:30,303 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:44:30,305 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:44:30,306 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:44:30,307 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:44:30,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:44:30,897 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:44:33,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:44:34,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:38,927 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:39,309 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:43,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:44,310 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:46,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:44:46,644 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:44:46,645 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:44:46,647 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:44:46,911 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:44:48,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:44:48,912 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:44:49,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:53,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:44:54,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:44:55,474 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:44:58,930 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:00,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:02,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:45:02,602 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:45:02,603 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:45:02,605 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:45:02,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:45:02,927 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:45:03,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:45:05,605 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:08,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:10,606 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:13,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:15,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:18,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:45:19,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:45:19,012 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:45:19,012 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:45:19,014 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:45:19,941 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:45:20,942 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:45:21,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:23,934 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:25,477 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:45:26,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:28,935 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:31,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:33,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:45:35,067 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:45:35,069 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:45:35,069 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:45:35,070 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:45:35,955 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:45:36,957 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:45:37,114 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:38,936 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:42,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:43,937 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:47,115 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:48,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:45:51,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:45:51,273 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:45:51,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:45:51,274 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:45:51,969 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:45:52,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:52,970 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:45:53,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:45:55,479 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:45:57,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:45:58,939 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:02,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:03,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:46:07,482 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:07,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:46:07,565 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:46:07,565 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:46:07,567 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:46:07,982 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:46:08,940 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:08,983 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:46:12,610 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:13,941 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:17,611 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:18,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:46:22,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:23,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:46:23,732 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:46:23,732 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:46:23,734 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:46:23,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:23,996 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:46:24,997 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:46:25,483 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:46:28,484 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:28,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:33,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:33,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:46:38,485 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:38,945 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:39,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:46:39,957 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:46:39,957 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:46:39,959 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:46:40,010 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:46:42,011 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:46:43,946 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:43,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:48,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:46:48,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:53,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:53,961 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:46:55,487 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:46:55,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:46:55,973 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:46:55,974 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:46:55,975 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:46:56,023 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:46:58,025 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:46:58,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:46:58,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:03,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:47:03,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:08,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:08,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:11,809 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:47:11,812 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:47:11,812 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:47:11,814 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:47:12,036 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:47:13,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:14,038 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:47:14,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:18,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:47:19,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:23,952 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:24,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:25,489 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:47:28,050 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:47:28,053 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:47:28,053 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:47:28,055 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:47:28,953 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:29,055 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:47:30,056 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:47:30,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:33,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:47:35,056 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:38,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:40,058 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:43,955 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:43,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:47:43,995 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:47:43,995 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:47:43,997 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:47:44,066 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:47:45,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:46,067 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:47:48,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:47:50,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:53,956 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:55,492 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:47:56,493 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:47:58,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:47:59,957 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:47:59,960 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:47:59,960 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:47:59,962 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:48:00,078 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:48:01,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:02,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:48:03,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:48:06,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:08,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:11,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:13,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:15,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:48:15,911 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:48:15,912 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:48:15,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:48:16,090 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:48:17,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:18,092 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:48:18,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:48:22,915 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:23,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:25,495 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:48:28,497 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:28,964 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:31,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:48:31,964 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:48:31,964 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:48:31,966 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:48:32,105 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:48:33,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:48:33,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:34,106 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:48:38,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:38,967 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:43,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:43,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:48,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:48:48,227 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:48:48,227 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:48:48,229 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:48:48,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:48:49,120 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:48:49,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:50,121 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:48:53,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:54,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:48:55,499 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:48:58,968 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:48:59,501 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:03,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:49:04,073 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:49:04,075 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:49:04,075 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:49:04,077 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:49:04,133 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:49:05,122 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:06,134 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:49:08,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:10,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:13,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:15,123 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:18,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:49:20,124 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:20,355 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:49:20,356 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:49:20,357 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:49:20,358 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:49:21,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:49:22,148 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:49:23,972 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:25,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:25,502 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:49:28,973 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:30,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:33,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:49:35,504 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:36,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:49:36,210 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:49:36,211 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:49:36,212 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:49:37,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:49:38,161 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:49:38,974 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:41,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:43,975 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:46,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:48,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:49:51,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:52,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:49:52,578 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:49:52,578 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:49:52,580 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:49:53,177 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:49:53,976 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:49:54,178 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:49:55,505 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:49:56,506 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:49:58,977 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:01,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:03,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:50:06,508 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:08,435 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:50:08,437 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:50:08,437 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:50:08,439 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:50:08,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:09,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:50:10,192 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:50:12,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:13,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:17,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:18,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:50:22,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:23,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:24,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:50:24,790 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:50:24,791 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:50:24,792 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:50:25,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:50:25,509 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:50:26,206 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:50:27,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:28,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:32,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:33,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:50:37,512 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:38,983 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:40,738 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:50:40,742 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:50:40,742 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:50:40,744 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:50:41,219 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:50:42,220 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:50:42,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:43,984 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:47,745 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:48,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:50:52,746 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:53,985 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:50:55,511 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:50:56,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:50:56,881 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:50:56,882 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:50:56,883 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:50:57,234 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:50:57,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:50:58,235 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:50:58,986 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:02,884 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:03,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:51:07,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:08,987 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:12,886 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:13,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:51:13,106 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:51:13,107 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:51:13,108 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:51:13,247 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:51:13,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:16,249 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:51:18,109 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:18,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:51:23,110 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:23,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:25,513 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:51:28,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:28,990 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:29,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:51:29,244 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:51:29,245 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:51:29,246 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:51:29,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:51:32,263 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:51:33,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:51:34,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:38,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:39,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:43,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:44,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:45,164 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:51:45,167 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:51:45,168 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:51:45,169 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:51:45,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:51:48,276 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:51:48,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:51:50,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:53,994 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:51:55,171 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:51:55,517 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:51:58,995 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:00,518 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:01,379 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:52:01,639 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:52:01,640 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:52:01,641 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:52:02,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:52:03,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:52:04,290 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:52:05,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:08,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:10,643 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:13,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:15,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:17,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:52:17,320 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:52:17,320 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:52:17,321 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:52:18,302 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:52:18,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:52:20,303 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:52:21,322 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:23,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:25,520 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:52:26,521 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:28,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:31,522 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:33,359 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:52:33,362 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:52:33,363 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:52:33,364 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:52:33,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:52:34,316 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:52:36,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:52:37,365 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:39,001 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:42,366 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:44,002 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:47,367 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:48,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:52:49,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:52:49,577 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:52:49,577 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:52:49,578 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:52:50,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:52:52,332 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:52:52,618 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:54,003 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:52:55,523 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:52:58,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:52:59,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:03,525 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:03,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:53:05,426 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:53:05,428 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:53:05,429 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:53:05,430 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:53:06,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:53:08,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:53:09,005 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:09,474 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:14,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:14,475 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:18,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:53:19,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:21,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:53:21,599 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:53:21,599 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:53:21,600 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:53:22,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:53:24,007 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:24,360 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:53:24,642 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:25,526 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:53:29,008 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:30,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:33,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:53:35,528 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:37,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:53:37,773 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:53:37,773 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:53:37,774 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:53:38,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:53:39,010 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:40,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:53:40,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:44,011 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:45,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:48,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:53:50,820 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:53,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:53:53,916 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:53:53,917 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:53:53,918 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:53:54,012 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:53:54,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:53:55,529 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:53:56,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:53:56,530 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:53:59,013 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:01,531 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:03,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:54:06,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:09,014 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:09,938 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:54:09,977 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:54:09,978 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:54:09,980 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:54:10,399 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:54:11,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:12,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:54:14,015 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:16,981 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:18,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:54:21,982 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:24,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:25,532 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:54:26,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:54:26,098 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:54:26,099 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:54:26,100 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:54:26,412 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:54:27,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:28,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:54:29,017 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:32,101 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:33,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:54:37,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:39,019 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:42,102 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:42,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:54:42,275 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:54:42,275 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:54:42,277 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:54:42,425 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:54:44,020 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:44,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:54:47,278 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:48,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:54:52,279 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:54,021 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:55,535 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:54:57,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:54:58,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:54:58,594 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:54:58,595 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:54:58,596 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:54:59,022 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:54:59,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:55:00,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:55:02,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:03,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:55:07,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:09,023 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:12,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:14,024 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:14,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:55:14,547 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:55:14,547 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:55:14,549 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:55:15,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:55:16,454 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:55:18,550 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:18,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:55:23,551 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:24,026 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:25,540 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:55:29,027 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:29,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:30,991 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:55:30,994 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:55:30,994 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:55:30,995 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:55:31,467 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:55:32,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:55:33,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:55:34,996 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:39,028 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:39,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:44,029 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:44,998 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:46,944 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:55:46,947 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:55:46,947 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:55:46,949 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:55:47,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:55:48,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:55:48,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:55:50,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:54,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:55:55,542 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:55:56,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:55:59,031 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:01,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:03,396 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:56:03,399 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:56:03,400 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:56:03,401 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:56:03,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:56:03,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:56:06,497 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:56:07,402 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:09,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:12,403 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:14,033 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:17,404 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:18,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:56:19,687 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:56:19,689 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:56:19,689 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:56:19,691 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:56:20,510 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:56:22,512 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:56:22,734 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:24,035 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:25,545 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:56:28,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:29,036 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:33,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:33,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:56:35,750 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:56:35,752 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:56:35,753 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:56:35,754 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:56:36,528 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:56:38,529 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:56:38,798 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:39,037 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:43,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:44,038 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:48,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:56:48,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:51,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:56:51,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:56:51,872 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:56:51,873 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:56:52,543 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:56:53,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:56:54,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:54,544 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:56:55,546 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:56:59,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:56:59,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:03,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:57:04,549 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:07,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:57:07,896 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:57:07,896 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:57:07,898 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:57:08,558 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:57:09,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:09,942 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:10,559 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:57:14,042 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:14,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:18,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:57:19,943 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:23,872 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:57:23,875 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:57:23,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:57:23,877 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:57:24,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:24,571 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:57:25,550 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:57:25,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:26,572 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:57:29,045 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:30,552 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:33,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:57:35,553 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:39,046 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:39,813 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:57:39,816 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:57:39,816 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:57:39,817 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:57:40,584 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:57:40,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:42,586 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:57:44,047 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:45,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:48,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:57:50,819 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:54,049 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:57:55,553 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:57:55,748 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:57:55,750 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:57:55,750 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:57:55,752 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:57:56,598 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:57:56,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:57:58,599 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:57:59,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:01,753 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:03,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:58:06,754 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:09,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:11,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:58:11,701 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:58:11,701 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:58:11,703 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:58:12,612 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:58:12,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:14,154 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:14,613 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:58:17,704 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:18,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:58:22,705 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:24,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:25,557 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:58:27,925 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:58:27,927 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:58:27,928 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:58:27,928 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:27,929 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:58:28,626 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:58:29,156 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:30,628 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:58:32,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:33,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:58:37,931 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:39,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:42,932 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:43,992 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:58:43,996 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:58:43,996 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:58:43,998 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:58:44,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:44,641 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:58:46,643 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:58:47,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:48,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:58:52,999 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:54,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:58:55,562 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:58:58,563 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:58:59,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:00,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:59:00,236 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:59:00,237 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:59:00,239 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:59:00,657 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:59:02,659 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:59:03,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:59:04,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:09,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:09,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:14,163 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:14,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:16,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:59:16,215 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:59:16,215 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:59:16,217 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:59:16,671 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:59:18,673 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:59:18,864 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:59:20,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:24,165 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:25,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:25,565 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:59:29,166 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:30,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:32,350 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:59:32,353 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:59:32,353 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:59:32,355 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:59:32,685 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:59:33,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:59:34,686 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:59:36,356 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:39,167 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:41,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:44,168 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:46,357 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:48,394 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 16:59:48,397 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 16:59:48,398 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 16:59:48,399 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 16:59:48,699 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 16:59:48,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 16:59:50,701 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 16:59:51,400 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:54,169 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 16:59:55,568 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 16:59:56,569 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 16:59:59,170 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:01,570 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:03,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:00:04,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:00:04,627 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:00:04,627 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:00:04,628 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:00:04,712 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:00:06,666 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:06,714 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:00:09,172 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:11,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:14,173 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:16,667 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:18,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:00:20,490 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:00:20,492 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:00:20,492 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:00:20,494 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:00:20,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:00:22,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:22,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:00:24,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:25,572 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:00:27,573 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:29,175 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:32,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:33,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:00:36,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:00:36,924 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:00:36,924 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:00:36,926 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:00:37,742 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:00:37,970 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:38,743 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:00:39,176 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:42,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:44,177 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:47,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:48,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:00:52,963 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:00:52,966 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:00:52,966 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:00:52,967 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:00:53,006 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:53,756 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:00:54,178 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:00:54,757 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:00:55,574 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:00:58,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:00:59,179 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:03,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:03,865 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:01:08,577 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:08,910 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:01:08,913 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:01:08,913 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:01:08,914 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:01:09,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:09,770 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:01:10,771 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:01:13,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:14,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:18,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:01:18,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:23,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:24,183 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:25,118 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:01:25,121 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:01:25,121 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:01:25,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:01:25,577 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:01:25,785 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:01:26,786 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:01:29,184 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:29,578 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:33,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:01:34,579 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:39,185 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:39,580 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:41,155 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:01:41,157 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:01:41,158 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:01:41,160 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:01:41,800 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:01:42,801 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:01:44,186 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:45,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:48,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:01:50,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:54,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:01:55,162 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:01:55,580 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:01:57,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:01:57,219 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:01:57,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:01:57,221 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:01:57,814 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:01:58,815 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:01:59,188 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:00,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:03,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:02:05,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:09,190 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:10,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:13,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:02:13,266 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:02:13,266 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:02:13,268 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:02:13,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:02:14,191 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:14,829 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:02:15,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:18,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:02:20,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:24,192 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:25,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:25,583 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:02:29,141 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:02:29,143 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:02:29,144 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:02:29,145 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:02:29,193 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:29,842 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:02:30,843 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:02:31,146 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:33,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:02:36,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:39,194 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:41,147 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:44,195 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:45,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:02:45,269 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:02:45,269 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:02:45,271 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:02:45,856 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:02:46,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:46,857 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:02:48,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:02:51,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:54,197 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:02:55,585 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:02:56,587 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:02:59,198 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:01,424 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:03:01,427 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:03:01,428 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:03:01,429 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:03:01,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:03:02,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:02,871 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:03:03,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:03:07,430 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:09,199 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:12,431 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:14,200 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:17,345 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:03:17,348 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:03:17,349 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:03:17,350 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:03:17,888 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:03:18,351 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:18,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:03:18,889 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:03:23,352 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:24,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:25,589 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:03:28,590 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:29,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:33,591 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:33,616 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:03:33,619 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:03:33,620 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:03:33,621 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:03:33,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:03:33,904 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:03:34,905 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:03:38,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:39,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:43,623 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:44,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:48,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:48,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:03:49,679 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:03:49,681 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:03:49,681 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:03:49,683 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:03:49,918 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:03:50,919 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:03:53,726 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:03:54,205 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:55,593 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:03:59,206 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:03:59,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:03,866 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:04:04,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:05,929 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:04:05,931 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:04:05,932 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:04:05,933 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:04:06,933 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:04:08,935 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:04:09,207 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:09,978 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:14,208 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:14,979 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:18,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:04:19,980 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:21,885 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:04:21,887 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:04:21,888 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:04:21,889 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:04:21,947 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:04:24,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:24,949 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:04:25,595 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:04:25,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:29,210 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:30,596 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:33,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:04:35,597 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:38,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:04:38,260 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:04:38,260 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:04:38,262 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:04:38,962 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:04:39,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:40,963 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:04:41,306 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:44,212 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:46,307 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:48,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:04:51,308 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:54,214 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:04:54,276 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:04:54,278 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:04:54,279 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:04:54,281 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:04:54,975 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:04:55,598 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:04:56,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:04:56,977 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:04:59,215 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:01,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:03,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:05:06,600 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:09,216 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:10,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:05:10,520 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:05:10,520 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:05:10,522 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:05:10,989 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:05:12,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:12,990 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:05:14,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:17,523 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:18,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:05:22,524 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:24,218 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:25,601 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:05:26,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:05:26,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:05:26,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:05:26,477 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:05:27,003 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:05:28,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:29,005 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:05:29,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:33,479 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:33,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:05:38,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:39,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:42,793 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:05:42,796 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:05:42,796 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:05:42,798 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:05:43,018 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:05:43,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:44,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:45,019 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:05:48,799 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:48,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:05:53,800 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:05:54,222 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:55,604 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:05:58,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:05:58,762 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:05:58,763 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:05:58,764 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:05:59,031 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:05:59,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:05:59,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:01,033 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:06:03,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:06:04,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:09,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:09,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:14,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:14,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:15,078 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:06:15,081 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:06:15,082 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:06:15,083 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:06:16,047 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:06:17,048 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:06:18,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:06:20,084 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:24,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:25,085 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:25,608 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:06:29,227 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:30,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:31,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:06:31,257 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:06:31,257 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:06:31,259 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:06:32,060 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:06:33,061 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:06:33,867 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:06:36,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:39,229 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:41,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:44,230 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:46,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:47,204 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:06:47,207 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:06:47,207 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:06:47,208 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:06:48,079 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:06:48,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:06:49,080 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:06:52,209 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:54,231 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:06:55,611 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:06:57,612 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:06:59,232 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:02,613 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:03,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:07:03,536 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:07:03,537 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:07:03,538 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:07:03,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:07:04,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:07:05,094 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:07:08,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:09,233 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:13,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:14,234 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:18,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:18,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:07:19,554 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:07:19,556 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:07:19,557 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:07:19,558 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:07:20,115 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:07:21,116 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:07:23,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:24,235 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:25,615 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:07:28,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:29,236 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:33,617 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:33,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:07:35,418 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:07:35,420 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:07:35,420 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:07:35,422 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:07:36,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:07:37,129 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:07:39,237 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:39,462 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:44,238 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:44,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:48,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:07:49,463 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:51,487 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:07:51,489 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:07:51,489 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:07:51,491 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:07:52,143 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:07:53,144 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:07:54,240 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:54,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:07:55,618 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:07:59,241 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:07:59,619 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:03,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:08:04,620 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:07,450 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:08:07,452 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:08:07,453 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:08:07,454 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:08:08,156 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:08:09,157 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:08:09,242 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:10,498 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:14,244 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:15,499 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:18,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:08:20,500 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:23,423 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:08:23,425 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:08:23,425 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:08:23,427 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:08:24,170 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:08:24,245 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:25,172 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:08:25,620 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:08:25,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:29,246 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:30,621 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:33,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:08:35,622 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:39,247 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:39,592 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:08:39,595 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:08:39,596 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:08:39,597 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:08:40,185 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:08:41,187 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:08:41,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:44,248 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:46,598 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:48,868 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:08:51,599 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:54,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:08:55,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:08:55,540 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:08:55,540 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:08:55,542 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:08:55,623 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:08:56,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:08:56,624 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:08:57,201 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:08:59,250 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:01,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:03,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:09:06,625 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:09,252 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:11,626 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:11,997 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:09:12,001 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:09:12,001 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:09:12,003 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:09:12,214 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:09:13,215 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:09:14,253 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:17,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:18,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:09:22,004 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:24,254 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:25,626 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:09:27,627 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:27,849 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:09:27,852 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:09:27,852 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:09:27,854 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:09:28,231 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:09:29,232 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:09:29,255 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:32,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:33,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:09:37,855 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:39,256 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:42,856 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:44,090 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:09:44,093 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:09:44,093 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:09:44,095 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:09:44,246 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:09:44,257 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:45,247 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:09:48,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:48,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:09:53,096 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:54,258 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:09:55,629 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:09:58,631 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:09:59,259 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:00,034 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:10:00,037 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:10:00,038 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:10:00,039 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:10:00,260 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:10:01,261 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:10:03,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:10:04,040 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:09,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:09,260 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:14,041 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:14,261 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:16,196 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:10:16,199 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:10:16,199 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:10:16,201 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:10:16,274 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:10:17,275 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:10:18,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:10:19,201 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:24,202 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:24,263 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:25,634 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:10:29,264 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:29,635 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:32,152 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:10:32,155 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:10:32,155 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:10:32,156 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:10:32,288 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:10:33,289 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:10:33,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:10:35,157 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:39,265 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:40,158 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:44,266 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:45,159 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:48,511 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:10:48,513 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:10:48,514 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:10:48,515 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:10:48,947 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:10:49,304 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:10:50,516 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:51,305 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:10:54,267 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:10:55,517 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:10:55,637 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:10:59,268 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:00,639 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:03,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:11:04,548 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:11:04,550 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:11:04,551 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:11:04,552 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:11:05,317 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:11:06,594 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:07,318 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:11:09,315 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:11,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:14,316 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:16,595 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:18,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:11:20,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:11:20,676 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:11:20,676 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:11:20,677 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:11:21,331 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:11:21,722 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:23,333 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:11:24,317 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:25,639 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:11:27,640 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:29,318 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:32,641 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:33,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:11:36,874 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:11:36,876 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:11:36,876 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:11:36,877 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:11:37,345 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:11:37,922 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:39,319 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:39,346 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:11:42,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:44,320 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:47,923 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:48,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:11:52,924 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:53,054 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:11:53,055 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:11:53,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:11:53,057 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:11:53,358 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:11:54,321 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:11:55,359 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:11:55,643 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:11:58,644 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:11:59,323 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:03,645 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:03,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:12:08,646 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:09,083 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:12:09,085 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:12:09,085 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:12:09,087 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:12:09,324 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:09,372 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:12:11,373 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:12:14,130 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:14,325 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:18,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:12:19,131 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:24,132 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:24,326 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:25,119 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:12:25,122 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:12:25,122 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:12:25,123 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:12:25,385 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:12:25,646 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:12:27,387 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:12:29,327 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:29,647 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:33,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:12:34,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:39,328 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:39,648 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:41,053 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:12:41,056 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:12:41,056 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:12:41,058 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:12:41,398 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:12:43,400 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:12:44,329 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:45,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:48,948 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:12:50,059 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:54,330 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:55,060 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:12:55,648 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:12:57,097 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:12:57,100 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:12:57,100 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:12:57,102 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:12:57,413 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:12:59,331 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:12:59,414 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:13:00,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:03,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:13:05,103 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:09,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:10,104 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:13,226 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:13:13,266 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:13:13,267 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:13:13,269 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:13:13,426 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:13:14,333 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:15,269 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:15,427 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:13:18,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:13:20,270 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:24,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:25,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:25,653 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:13:29,174 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:13:29,177 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:13:29,178 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:13:29,179 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:13:29,335 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:29,440 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:13:31,180 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:31,441 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:13:33,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:13:36,181 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:39,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:41,182 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:44,337 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:45,380 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:13:45,383 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:13:45,383 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:13:45,385 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:13:45,453 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:13:46,385 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:47,455 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:13:48,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:13:51,386 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:54,339 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:13:55,656 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:13:56,657 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:13:59,340 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:01,601 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:14:01,604 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:14:01,605 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:14:01,606 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:14:02,468 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:14:02,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:03,469 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:14:03,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:14:07,607 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:09,341 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:12,608 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:14,342 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:17,609 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:17,774 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:14:17,777 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:14:17,778 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:14:17,779 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:14:18,481 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:14:18,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:14:19,482 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:14:22,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:24,343 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:25,661 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:14:28,662 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:29,344 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:33,663 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:33,724 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:14:33,727 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:14:33,727 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:14:33,729 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:14:33,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:14:34,495 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:14:35,496 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:14:38,730 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:39,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:43,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:44,346 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:48,731 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:48,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:14:49,869 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:14:49,871 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:14:49,871 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:14:49,872 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:14:50,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:14:51,509 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:14:53,914 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:14:54,348 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:55,662 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:14:59,349 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:14:59,664 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:03,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:15:04,665 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:05,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:15:05,911 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:15:05,911 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:15:05,913 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:15:06,523 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:15:07,524 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:15:09,756 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:09,958 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:14,758 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:14,959 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:18,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:15:19,960 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:22,567 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:15:22,569 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:15:22,569 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:15:22,571 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:15:23,539 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:15:23,540 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:15:24,759 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:25,614 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:25,669 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:15:29,760 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:30,670 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:33,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:15:35,671 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:38,527 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:15:38,529 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:15:38,529 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:15:38,530 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:15:38,552 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:15:39,553 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:15:39,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:41,574 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:44,762 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:46,575 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:48,949 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:15:51,576 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:54,764 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:15:55,044 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:15:55,046 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:15:55,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:15:55,048 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:15:55,567 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:15:55,671 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:15:56,672 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:15:57,569 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:15:59,765 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:01,673 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:03,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:16:06,674 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:09,766 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:10,888 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:16:10,891 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:16:10,892 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:16:10,893 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:16:11,581 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:16:11,894 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:13,582 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:16:14,767 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:16,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:18,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:16:21,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:24,768 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:25,672 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:16:27,217 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:16:27,220 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:16:27,221 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:16:27,221 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:27,222 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:16:27,595 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:16:29,597 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:16:29,769 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:32,223 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:33,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:16:37,224 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:39,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:42,225 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:43,507 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:16:43,510 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:16:43,510 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:16:43,512 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:16:43,609 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:16:44,771 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:45,610 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:16:47,513 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:48,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:16:52,514 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:54,772 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:16:55,675 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:16:57,676 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:16:59,470 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:16:59,473 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:16:59,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:16:59,475 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:16:59,624 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:16:59,773 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:01,625 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:17:03,476 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:03,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:17:08,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:09,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:13,477 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:14,775 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:15,890 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:17:15,893 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:17:15,893 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:17:15,895 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:17:16,637 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:17:17,638 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:17:18,895 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:18,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:17:23,896 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:24,777 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:25,678 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:17:29,680 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:29,778 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:31,843 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:17:31,848 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:17:31,848 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:17:31,849 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:17:32,651 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:17:33,652 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:17:33,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:17:34,850 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:39,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:39,851 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:44,780 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:44,852 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:47,723 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:17:47,726 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:17:47,726 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:17:47,727 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:17:48,666 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:17:48,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:17:49,667 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:17:50,728 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:54,782 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:17:55,681 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:17:56,682 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:17:59,783 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:01,683 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:03,870 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:18:03,873 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:18:03,873 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:18:03,875 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:18:03,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:18:04,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:18:05,680 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:06,876 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:09,784 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:11,877 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:14,785 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:16,878 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:18,950 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:18:19,725 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:18:19,727 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:18:19,727 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:18:19,728 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:18:20,694 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:18:21,695 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:22,770 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:24,786 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:25,684 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:18:28,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:29,787 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:33,686 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:33,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:18:35,858 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:18:35,860 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:18:35,860 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:18:35,861 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:18:36,707 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:18:37,708 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:38,906 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:39,788 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:43,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:44,789 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:48,907 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:48,951 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:18:51,909 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: partial_history +2024-08-29 17:18:51,911 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:18:51,911 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:18:51,912 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:18:52,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:18:53,722 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:53,954 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:54,790 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:18:55,687 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:18:57,725 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:59,488 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:18:59,727 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:18:59,993 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:02,729 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:19:03,730 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:19:04,039 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:19:04,971 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:07,733 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:19:09,532 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:10,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:14,533 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:15,655 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:18,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:19:20,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:24,534 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:25,656 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:25,691 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:19:29,535 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:30,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:33,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:19:35,693 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:39,536 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:40,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:44,537 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:45,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:48,965 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:19:50,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:54,538 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:19:55,692 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:19:56,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:19:59,539 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:01,694 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:03,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:20:06,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:09,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:11,695 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:14,540 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:16,696 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:18,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:20:21,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:24,541 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:25,696 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:20:26,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:29,542 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:31,697 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:33,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:20:36,698 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:39,543 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:41,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:44,544 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:46,699 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:48,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:20:51,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:54,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:20:55,699 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:20:57,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:20:59,545 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:21:02,700 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:03,966 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:21:07,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:09,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:21:12,701 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:14,547 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: keepalive +2024-08-29 17:21:17,271 DEBUG SenderThread:12190 [sender.py:send():382] send: exit +2024-08-29 17:21:17,271 INFO SenderThread:12190 [sender.py:send_exit():589] handling exit code: 1 +2024-08-29 17:21:17,271 INFO SenderThread:12190 [sender.py:send_exit():591] handling runtime: 71460 +2024-08-29 17:21:17,272 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:21:17,273 INFO SenderThread:12190 [sender.py:send_exit():597] send defer +2024-08-29 17:21:17,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:17,273 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 0 +2024-08-29 17:21:17,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:17,273 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-08-29 17:21:17,273 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 1 +2024-08-29 17:21:17,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:17,273 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 1 +2024-08-29 17:21:17,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:17,273 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-08-29 17:21:17,273 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 2 +2024-08-29 17:21:17,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:17,274 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 2 +2024-08-29 17:21:17,274 INFO HandlerThread:12190 [system_monitor.py:finish():203] Stopping system monitor +2024-08-29 17:21:17,274 INFO HandlerThread:12190 [interfaces.py:finish():202] Joined cpu monitor +2024-08-29 17:21:17,274 DEBUG SystemMonitor:12190 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-08-29 17:21:17,275 INFO HandlerThread:12190 [interfaces.py:finish():202] Joined disk monitor +2024-08-29 17:21:17,275 DEBUG SystemMonitor:12190 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-08-29 17:21:17,826 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:21:18,471 INFO HandlerThread:12190 [interfaces.py:finish():202] Joined gpu monitor +2024-08-29 17:21:18,471 INFO HandlerThread:12190 [interfaces.py:finish():202] Joined memory monitor +2024-08-29 17:21:18,471 INFO HandlerThread:12190 [interfaces.py:finish():202] Joined network monitor +2024-08-29 17:21:18,471 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:18,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:18,473 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,473 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-08-29 17:21:18,473 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 3 +2024-08-29 17:21:18,473 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:18,474 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 3 +2024-08-29 17:21:18,475 DEBUG SenderThread:12190 [sender.py:send():382] send: stats +2024-08-29 17:21:18,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:18,476 DEBUG SenderThread:12190 [sender.py:send():382] send: history +2024-08-29 17:21:18,476 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: summary_record +2024-08-29 17:21:18,478 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:21:18,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,478 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-08-29 17:21:18,478 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 4 +2024-08-29 17:21:18,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:18,478 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 4 +2024-08-29 17:21:18,478 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,478 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-08-29 17:21:18,478 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 5 +2024-08-29 17:21:18,478 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:18,478 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 5 +2024-08-29 17:21:18,479 DEBUG SenderThread:12190 [sender.py:send():382] send: summary +2024-08-29 17:21:18,480 INFO SenderThread:12190 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-08-29 17:21:18,480 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,480 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-08-29 17:21:18,480 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 6 +2024-08-29 17:21:18,480 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:18,480 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 6 +2024-08-29 17:21:18,481 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,481 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-08-29 17:21:18,481 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 7 +2024-08-29 17:21:18,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:18,481 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:18,481 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 7 +2024-08-29 17:21:18,481 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:18,481 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-08-29 17:21:18,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:21:18,827 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:21:19,271 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:20,066 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 8 +2024-08-29 17:21:20,066 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:20,066 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:20,066 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 8 +2024-08-29 17:21:20,067 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:20,067 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-08-29 17:21:20,067 INFO SenderThread:12190 [job_builder.py:build():296] Attempting to build job artifact +2024-08-29 17:21:20,069 INFO SenderThread:12190 [job_builder.py:_get_source_type():426] is repo sourced job +2024-08-29 17:21:20,149 INFO SenderThread:12190 [job_builder.py:build():402] adding wandb-job metadata file +2024-08-29 17:21:20,161 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 9 +2024-08-29 17:21:20,161 DEBUG SenderThread:12190 [sender.py:send():382] send: artifact +2024-08-29 17:21:20,161 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:20,162 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 9 +2024-08-29 17:21:20,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:20,828 INFO Thread-12 :12190 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:21:21,047 INFO SenderThread:12190 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTE2NDgyMTAwMw==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQxNjgwMzg3NA==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTE2NDgyMTAwMw==', 'versionIndex': 1}}} +2024-08-29 17:21:21,047 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:21,047 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-08-29 17:21:21,047 INFO SenderThread:12190 [dir_watcher.py:finish():358] shutting down directory watcher +2024-08-29 17:21:21,829 INFO SenderThread:12190 [dir_watcher.py:finish():388] scan: /project/wandb/run-20240828_213016-pkqwjpaw/files +2024-08-29 17:21:21,830 INFO SenderThread:12190 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt requirements.txt +2024-08-29 17:21:21,830 INFO SenderThread:12190 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml config.yaml +2024-08-29 17:21:21,830 INFO SenderThread:12190 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-metadata.json wandb-metadata.json +2024-08-29 17:21:21,830 INFO SenderThread:12190 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json wandb-summary.json +2024-08-29 17:21:21,833 INFO SenderThread:12190 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log output.log +2024-08-29 17:21:21,835 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 10 +2024-08-29 17:21:21,835 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:21,835 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:21,837 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 10 +2024-08-29 17:21:21,837 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:21,837 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-08-29 17:21:21,837 INFO SenderThread:12190 [file_pusher.py:finish():172] shutting down file pusher +2024-08-29 17:21:22,272 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:22,273 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:22,337 INFO wandb-upload_0:12190 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240828_213016-pkqwjpaw/files/requirements.txt +2024-08-29 17:21:22,338 INFO wandb-upload_2:12190 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240828_213016-pkqwjpaw/files/wandb-summary.json +2024-08-29 17:21:22,344 INFO wandb-upload_1:12190 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240828_213016-pkqwjpaw/files/config.yaml +2024-08-29 17:21:23,018 INFO wandb-upload_3:12190 [upload_job.py:push():131] Uploaded file /project/wandb/run-20240828_213016-pkqwjpaw/files/output.log +2024-08-29 17:21:23,219 INFO Thread-11 (_thread_body):12190 [sender.py:transition_state():617] send defer: 11 +2024-08-29 17:21:23,219 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:23,219 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 11 +2024-08-29 17:21:23,219 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:23,219 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-08-29 17:21:23,219 INFO SenderThread:12190 [file_pusher.py:join():178] waiting for file pusher +2024-08-29 17:21:23,220 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 12 +2024-08-29 17:21:23,220 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:23,220 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 12 +2024-08-29 17:21:23,220 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:23,220 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-08-29 17:21:23,220 INFO SenderThread:12190 [file_stream.py:finish():595] file stream finish called +2024-08-29 17:21:23,273 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:23,813 INFO SenderThread:12190 [file_stream.py:finish():599] file stream finish is done +2024-08-29 17:21:23,813 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 13 +2024-08-29 17:21:23,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:23,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:23,814 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 13 +2024-08-29 17:21:23,814 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: status_report +2024-08-29 17:21:23,814 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:23,814 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-08-29 17:21:23,814 INFO SenderThread:12190 [sender.py:transition_state():617] send defer: 14 +2024-08-29 17:21:23,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: defer +2024-08-29 17:21:23,815 DEBUG SenderThread:12190 [sender.py:send():382] send: final +2024-08-29 17:21:23,815 INFO HandlerThread:12190 [handler.py:handle_request_defer():172] handle defer: 14 +2024-08-29 17:21:23,815 DEBUG SenderThread:12190 [sender.py:send():382] send: footer +2024-08-29 17:21:23,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: defer +2024-08-29 17:21:23,815 INFO SenderThread:12190 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-08-29 17:21:23,815 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:23,815 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:23,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: poll_exit +2024-08-29 17:21:23,816 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: server_info +2024-08-29 17:21:23,816 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: poll_exit +2024-08-29 17:21:23,817 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: server_info +2024-08-29 17:21:23,817 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: get_summary +2024-08-29 17:21:23,818 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: sampled_history +2024-08-29 17:21:23,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: internal_messages +2024-08-29 17:21:23,823 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: job_info +2024-08-29 17:21:23,985 DEBUG SenderThread:12190 [sender.py:send_request():409] send_request: job_info +2024-08-29 17:21:23,986 INFO MainThread:12190 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-08-29 17:21:23,987 INFO MainThread:12190 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-08-29 17:21:23,988 INFO MainThread:12190 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-08-29 17:21:23,988 DEBUG HandlerThread:12190 [handler.py:handle_request():146] handle_request: shutdown +2024-08-29 17:21:23,988 INFO HandlerThread:12190 [handler.py:finish():869] shutting down handler +2024-08-29 17:21:24,824 INFO WriterThread:12190 [datastore.py:close():296] close: /project/wandb/run-20240828_213016-pkqwjpaw/run-pkqwjpaw.wandb +2024-08-29 17:21:24,986 INFO SenderThread:12190 [sender.py:finish():1572] shutting down sender +2024-08-29 17:21:24,986 INFO SenderThread:12190 [file_pusher.py:finish():172] shutting down file pusher +2024-08-29 17:21:24,986 INFO SenderThread:12190 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20240828_213016-pkqwjpaw/logs/debug.log b/wandb/run-20240828_213016-pkqwjpaw/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..db77a38642d1cade69443fb5e3d735a7f4c6e603 --- /dev/null +++ b/wandb/run-20240828_213016-pkqwjpaw/logs/debug.log @@ -0,0 +1,30 @@ +2024-08-28 21:30:16,381 INFO MainThread:11552 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Configure stats pid to 11552 +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'Train sample'} +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20240828_213016-pkqwjpaw/logs/debug.log +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20240828_213016-pkqwjpaw/logs/debug-internal.log +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:init():566] calling init triggers +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'FULL_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['1754785366', '/project/datas/llm-jp-corpus-v2/ja-wiki/data/data_text_document', '28623823675', '/project/datas/llm-jp-corpus-v2/ja-cc/level0/data_text_document'], 'valid_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'test_data_path': ['1205770', '/work/llm_recipes/datasets/bin/baseline/llm_jp_corpus_v2_ja_wiki_validation_0/data_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/Qwen/Qwen2-1.5B', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'yans-baseline-qwen2-1.5B-3.5e-5_train_2024-08-28-21:29:36', 'wandb_project': 'yans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', 'save': '/work/llm_recipes/models/yans-baseline-qwen2-1.5B-3.5e-5', 'base_model': '/share/pretrained_lm/Qwen/Qwen2-1.5B', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 3.5e-05, 'lr_decay_style': 'cosine', 'lr_decay_iters': 23178, 'lr_warmup_iters': 500, 'min_lr': 3.5e-06, 'train_iters': 23178, 'train_samples': None, 'global_batch_size': 1280, 'micro_batch_size': 16, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/yans-baseline-qwen2-1.5B-3.5e-5', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 8, 'padded_vocab_size': 151680, 'gradient_accumulation_steps': 10} +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:init():616] starting backend +2024-08-28 21:30:16,382 INFO MainThread:11552 [wandb_init.py:init():620] setting up manager +2024-08-28 21:30:16,388 INFO MainThread:11552 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-08-28 21:30:16,389 INFO MainThread:11552 [wandb_init.py:init():628] backend started and connected +2024-08-28 21:30:16,396 INFO MainThread:11552 [wandb_init.py:init():720] updated telemetry +2024-08-28 21:30:16,488 INFO MainThread:11552 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-08-28 21:30:16,923 INFO MainThread:11552 [wandb_run.py:_on_init():2262] communicating current version +2024-08-28 21:30:16,947 INFO MainThread:11552 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.17.7 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-08-28 21:30:16,947 INFO MainThread:11552 [wandb_init.py:init():804] starting run threads in backend +2024-08-28 21:30:17,125 INFO MainThread:11552 [wandb_run.py:_console_start():2241] atexit reg +2024-08-28 21:30:17,125 INFO MainThread:11552 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-08-28 21:30:17,126 INFO MainThread:11552 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-08-28 21:30:17,126 INFO MainThread:11552 [wandb_run.py:_redirect():2186] Redirects installed. +2024-08-28 21:30:17,126 INFO MainThread:11552 [wandb_init.py:init():847] run started, returning control to user process +2024-08-28 21:31:23,644 INFO MainThread:11552 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'Qwen2ForCausalLM', 'activation_function': 'silu', 'hidden_size': 1536, 'model_type': 'qwen2', 'max_position_embeddings': 1024, 'num_attention_heads': 12, 'num_hidden_layers': 28} +2024-08-28 21:31:23,644 INFO MainThread:11552 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 8} +2024-08-29 17:21:24,988 WARNING MsgRouterThr:11552 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml b/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64cc81c3d2cd7089210e4948a821ac5a9eac5d90 --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml @@ -0,0 +1,340 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: NO_SHARD +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '183190553' + - /project/trans/datasets/train/documents_text_document +valid_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +test_data_path: + desc: null + value: + - '61085876' + - /project/trans/datasets/valid/documents_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-20:26:21 +wandb_project: + desc: null + value: trans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +save: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs +base_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 200 +save_interval: + desc: null + value: 200 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 0.00015 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 50 +lr_warmup_iters: + desc: null + value: 5 +min_lr: + desc: null + value: 1.5e-05 +train_iters: + desc: null + value: 50 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 256 +micro_batch_size: + desc: null + value: 1 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 10 +hf_repo_id: + desc: null + value: koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 1 +padded_vocab_size: + desc: null + value: 99584 +gradient_accumulation_steps: + desc: null + value: 256 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1730374002.66897 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: LlamaForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 3072 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 24 +num_hidden_layers: + desc: null + value: 28 diff --git a/wandb/run-20241031_202642-fuo0pr3i/files/output.log b/wandb/run-20241031_202642-fuo0pr3i/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d310986e35a13eebeae2e9487ce497b6b1986490 --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/files/output.log @@ -0,0 +1,267 @@ +Created Hugging Face repository with ID koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` + +Loading checkpoint shards: 50%|█████ | 1/2 [01:19<01:19, 79.47s/it] +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs, skipping model loading +--> Model /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct + +Loading checkpoint shards: 100%|██████████| 2/2 [02:01<00:00, 60.89s/it] +BFloat16 enabled for mixed precision - using bfSixteen policy +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 12800 + validation: 2560 + test: 2560 +> building train, validation, and test datasets for GPT ... +Let split = None +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs, skipping optimizer loading +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(99584, 3072) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=3072, out_features=3072, bias=False) + (k_proj): Linear(in_features=3072, out_features=3072, bias=False) + (v_proj): Linear(in_features=3072, out_features=3072, bias=False) + (o_proj): Linear(in_features=3072, out_features=3072, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=3072, out_features=8192, bias=False) + (up_proj): Linear(in_features=3072, out_features=8192, bias=False) + (down_proj): Linear(in_features=8192, out_features=3072, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=3072, out_features=99584, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 24, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 99584 +} +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 83.46563044499827, Tokens per sec: 3807.931090820854, Loss: 3.1152522563934326 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 90.34626412279788, Tokens per sec: 4121.844479679852, Loss: 2.919541835784912 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 90.72878683166805, Tokens per sec: 4139.296215301876, Loss: 3.4609086513519287 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 90.5697147903784, Tokens per sec: 4132.038912284123, Loss: 2.9889941215515137 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 89.90641358915883, Tokens per sec: 4101.777291384162, Loss: 3.0124733448028564 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 90.50600451170307, Tokens per sec: 4129.132274549775, Loss: 2.9347522258758545 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 90.41470968161596, Tokens per sec: 4124.967153887935, Loss: 2.9396538734436035 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 90.2499320674706, Tokens per sec: 4117.449546980396, Loss: 2.8515565395355225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 9 , TFLOPS: 90.96883938255716, Tokens per sec: 4150.2480713783, Loss: 2.7779500484466553 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 10 , TFLOPS: 90.89030666397508, Tokens per sec: 4146.665193262588, Loss: 2.7331361770629883 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 11 , TFLOPS: 90.54630159120393, Tokens per sec: 4130.9707378919, Loss: 2.7144951820373535 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 12 , TFLOPS: 90.804596783966, Tokens per sec: 4142.754873348427, Loss: 2.668790340423584 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 13 , TFLOPS: 90.82671703162298, Tokens per sec: 4143.764059744559, Loss: 2.6441712379455566 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 14 , TFLOPS: 90.78333482765136, Tokens per sec: 4141.784844558487, Loss: 2.613452672958374 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 15 , TFLOPS: 90.50907219553717, Tokens per sec: 4129.2722307040185, Loss: 2.611032247543335 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 16 , TFLOPS: 90.6771316902523, Tokens per sec: 4136.939566008639, Loss: 2.571103572845459 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 17 , TFLOPS: 90.87770225904788, Tokens per sec: 4146.090145723288, Loss: 2.57216477394104 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 18 , TFLOPS: 91.13480972726933, Tokens per sec: 4157.820093927152, Loss: 2.566211223602295 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 19 , TFLOPS: 90.2632917270603, Tokens per sec: 4118.059051309821, Loss: 2.511382818222046 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 20 , TFLOPS: 90.27349125126283, Tokens per sec: 4118.524381591463, Loss: 2.5154967308044434 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 21 , TFLOPS: 91.11009858574018, Tokens per sec: 4156.692703843151, Loss: 2.5131828784942627 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 22 , TFLOPS: 90.76844844388178, Tokens per sec: 4141.105686882641, Loss: 2.5021755695343018 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 23 , TFLOPS: 90.22323058566491, Tokens per sec: 4116.231352111481, Loss: 2.496105194091797 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 24 , TFLOPS: 90.3810065941473, Tokens per sec: 4123.42952655625, Loss: 2.4846913814544678 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 25 , TFLOPS: 90.89127980610279, Tokens per sec: 4146.70959067676, Loss: 2.476154327392578 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 26 , TFLOPS: 90.79510204334753, Tokens per sec: 4142.321697227819, Loss: 2.4694666862487793 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 27 , TFLOPS: 90.69610818629873, Tokens per sec: 4137.805325829835, Loss: 2.4508180618286133 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 28 , TFLOPS: 90.30850080031388, Tokens per sec: 4120.121613285471, Loss: 2.449306011199951 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 29 , TFLOPS: 91.09708146377791, Tokens per sec: 4156.098827020214, Loss: 2.452608108520508 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 30 , TFLOPS: 90.69698164120638, Tokens per sec: 4137.845175239485, Loss: 2.4541070461273193 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 31 , TFLOPS: 91.01731698404008, Tokens per sec: 4152.459752580623, Loss: 2.4211976528167725 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 32 , TFLOPS: 90.73720326569989, Tokens per sec: 4139.680196117128, Loss: 2.4361917972564697 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 33 , TFLOPS: 90.39105964245766, Tokens per sec: 4123.888174205761, Loss: 2.435288906097412 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 34 , TFLOPS: 90.59987936351307, Tokens per sec: 4133.415102882199, Loss: 2.43648624420166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 35 , TFLOPS: 90.92715645479835, Tokens per sec: 4148.346381835874, Loss: 2.428072452545166 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 36 , TFLOPS: 90.97257135644028, Tokens per sec: 4150.418334267394, Loss: 2.4085240364074707 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 37 , TFLOPS: 90.16423698166008, Tokens per sec: 4113.5399020181685, Loss: 2.427152156829834 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 38 , TFLOPS: 90.75295475413623, Tokens per sec: 4140.398822241721, Loss: 2.4176218509674072 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 39 , TFLOPS: 90.46842287901892, Tokens per sec: 4127.417697342619, Loss: 2.4117372035980225 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 40 , TFLOPS: 90.32624311205915, Tokens per sec: 4120.931066232243, Loss: 2.392359733581543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 41 , TFLOPS: 90.45822613346098, Tokens per sec: 4126.952493830368, Loss: 2.398505210876465 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 42 , TFLOPS: 90.23121984568967, Tokens per sec: 4116.595844076362, Loss: 2.4123034477233887 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 43 , TFLOPS: 91.02231549739636, Tokens per sec: 4152.68779847587, Loss: 2.428980827331543 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 44 , TFLOPS: 89.8794984637352, Tokens per sec: 4100.549349507158, Loss: 2.3931190967559814 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 45 , TFLOPS: 91.03697749826263, Tokens per sec: 4153.35671918796, Loss: 2.410999059677124 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 46 , TFLOPS: 90.29296806025584, Tokens per sec: 4119.412967062142, Loss: 2.4177074432373047 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 47 , TFLOPS: 91.11362585767964, Tokens per sec: 4156.853627667873, Loss: 2.4054439067840576 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 48 , TFLOPS: 90.80597749169735, Tokens per sec: 4142.817865023797, Loss: 2.405735731124878 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 49 , TFLOPS: 90.75304533743876, Tokens per sec: 4140.402954900544, Loss: 2.3881704807281494 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 50 , TFLOPS: 90.69643934171684, Tokens per sec: 4137.820434048701, Loss: 2.394106388092041 +------------------------------------------------------------------ +Saving checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050 +Saving model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/model.pt +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:773: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +/usr/local/lib/python3.10/dist-packages/torch/distributed/fsdp/_state_dict_utils.py:716: UserWarning: When using ``NO_SHARD`` for ``ShardingStrategy``, full_state_dict willbe returned. + warnings.warn( +Saved model state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/model.pt +Saving optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/optimizer.pt +[rank0]:[2024-10-31 21:22:18,591] torch.distributed.fsdp._debug_utils: [WARNING] FSDP _optim_state_dict() profiling: defaultdict(, {'preprocessing': 0.0076531579998118104, 'preprocessing_with_comm': 0.000735907000489533, 'state_converting': 7.802827932999207, : 7.812808361000862}) +Saved optimizer state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/optimizer.pt +Saving scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/scheduler.pt +Saved scheduler state dict to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/scheduler.pt +Saving RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/rng.pt +Saved RNG states to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050/rng.pt +Saved checkpoint to /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs/iter_0000050, took 30.33s \ No newline at end of file diff --git a/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt b/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20241031_202642-fuo0pr3i/files/wandb-metadata.json b/wandb/run-20241031_202642-fuo0pr3i/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5e5ee816773585b2e57506b34ac4333ea40ac3cd --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/files/wandb-metadata.json @@ -0,0 +1,220 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-10-31T11:26:43.313615", + "startedAt": "2024-10-31T11:26:42.655841", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "1", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "256", + "--train-iters", + "50", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct", + "--train-data-path", + "183190553", + "/project/trans/datasets/train/documents_text_document", + "--valid-data-path", + "61085876", + "/project/trans/datasets/valid/documents_text_document", + "--test-data-path", + "61085876", + "/project/trans/datasets/valid/documents_text_document", + "--lr", + "1.5e-4", + "--min-lr", + "1.5e-5", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "5", + "--lr-decay-iters", + "50", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "200", + "--eval-interval", + "200", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct", + "--save", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--load", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "NO_SHARD", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "10", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "trans_experiment", + "--wandb-name", + "llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-20:26:21" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 18, + "cpu_count_logical": 18, + "cpu_freq": { + "current": 2400.0319999999997, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 56.48782730102539 + } +} diff --git a/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json b/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2246f6b5507f2b6d3b8df5a0a01d4986da41f7ad --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.394106388092041, "training/perplexity": 10.958401124666537, "utils/batch_size": 1, "utils/global_batch_size": 256, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 256, "utils/iteration": 50, "optimizer/lr": 1.5e-05, "optimizer/variance_l2": 0.014598608131742317, "optimizer/variance_sqrt_l2": 0.6901579110192728, "optimizer/momentum_l2": 0.16585301682175427, "optimizer/weight_l2": 1144.370569352428, "optimizer/variance_l1": 0.477020263671875, "optimizer/variance_sqrt_l1": 13762.0, "optimizer/momentum_l1": 2909.5, "optimizer/weight_l1": 51806208.0, "optimizer/variance_abs_max": 0.006744384765625, "optimizer/variance_sqrt_abs_max": 0.08203125, "optimizer/momentum_abs_max": 0.01495361328125, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 63.41502831800062, "stats/tokens_per_sec": 4137.820434048701, "stats/tokens_per_sec_per_gpu": 4137.820434048701, "stats/tflops": 90.69643934171684, "_timestamp": 1730377322.4009933, "_runtime": 3319.7320232391357, "_step": 50, "_wandb": {"runtime": 3354}} \ No newline at end of file diff --git a/wandb/run-20241031_202642-fuo0pr3i/logs/debug-internal.log b/wandb/run-20241031_202642-fuo0pr3i/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a80e6ac29baea1f5adcee6bb7604a50a3cc34b91 --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/logs/debug-internal.log @@ -0,0 +1,1921 @@ +2024-10-31 20:26:42,671 INFO StreamThr :10220 [internal.py:wandb_internal():86] W&B internal server running at pid: 10220, started at: 2024-10-31 20:26:42.670144 +2024-10-31 20:26:42,672 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status +2024-10-31 20:26:42,673 INFO WriterThread:10220 [datastore.py:open_for_write():87] open: /project/wandb/run-20241031_202642-fuo0pr3i/run-fuo0pr3i.wandb +2024-10-31 20:26:42,674 DEBUG SenderThread:10220 [sender.py:send():382] send: header +2024-10-31 20:26:42,776 DEBUG SenderThread:10220 [sender.py:send():382] send: run +2024-10-31 20:26:43,207 INFO SenderThread:10220 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20241031_202642-fuo0pr3i/files +2024-10-31 20:26:43,207 INFO SenderThread:10220 [sender.py:_start_run_threads():1136] run started: fuo0pr3i with start time 1730374002.66897 +2024-10-31 20:26:43,212 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: check_version +2024-10-31 20:26:43,212 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: check_version +2024-10-31 20:26:43,294 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: run_start +2024-10-31 20:26:43,300 DEBUG HandlerThread:10220 [system_info.py:__init__():27] System info init +2024-10-31 20:26:43,300 DEBUG HandlerThread:10220 [system_info.py:__init__():42] System info init done +2024-10-31 20:26:43,300 INFO HandlerThread:10220 [system_monitor.py:start():194] Starting system monitor +2024-10-31 20:26:43,301 INFO SystemMonitor:10220 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-10-31 20:26:43,301 INFO HandlerThread:10220 [system_monitor.py:probe():214] Collecting system info +2024-10-31 20:26:43,301 INFO SystemMonitor:10220 [interfaces.py:start():190] Started cpu monitoring +2024-10-31 20:26:43,301 INFO SystemMonitor:10220 [interfaces.py:start():190] Started disk monitoring +2024-10-31 20:26:43,302 INFO SystemMonitor:10220 [interfaces.py:start():190] Started gpu monitoring +2024-10-31 20:26:43,303 INFO SystemMonitor:10220 [interfaces.py:start():190] Started memory monitoring +2024-10-31 20:26:43,304 INFO SystemMonitor:10220 [interfaces.py:start():190] Started network monitoring +2024-10-31 20:26:43,313 DEBUG HandlerThread:10220 [system_info.py:probe():151] Probing system +2024-10-31 20:26:43,315 DEBUG HandlerThread:10220 [system_info.py:_probe_git():136] Probing git +2024-10-31 20:26:43,327 DEBUG HandlerThread:10220 [system_info.py:_probe_git():144] Probing git done +2024-10-31 20:26:43,328 DEBUG HandlerThread:10220 [system_info.py:probe():199] Probing system done +2024-10-31 20:26:43,328 DEBUG HandlerThread:10220 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-10-31T11:26:43.313615', 'startedAt': '2024-10-31T11:26:42.655841', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '1', '--valid_micro_batch_size', '1', '--global-batch-size', '256', '--train-iters', '50', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct', '--train-data-path', '183190553', '/project/trans/datasets/train/documents_text_document', '--valid-data-path', '61085876', '/project/trans/datasets/valid/documents_text_document', '--test-data-path', '61085876', '/project/trans/datasets/valid/documents_text_document', '--lr', '1.5e-4', '--min-lr', '1.5e-5', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '5', '--lr-decay-iters', '50', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '200', '--eval-interval', '200', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct', '--save', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', '--load', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'NO_SHARD', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '10', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'trans_experiment', '--wandb-name', 'llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-20:26:21'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 18, 'cpu_count_logical': 18, 'cpu_freq': {'current': 2400.0319999999997, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 56.48782730102539}} +2024-10-31 20:26:43,328 INFO HandlerThread:10220 [system_monitor.py:probe():224] Finished collecting system info +2024-10-31 20:26:43,328 INFO HandlerThread:10220 [system_monitor.py:probe():227] Publishing system info +2024-10-31 20:26:43,329 INFO HandlerThread:10220 [system_monitor.py:probe():229] Finished publishing system info +2024-10-31 20:26:43,334 DEBUG SenderThread:10220 [sender.py:send():382] send: files +2024-10-31 20:26:43,334 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-10-31 20:26:43,346 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: python_packages +2024-10-31 20:26:43,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:26:43,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:26:43,347 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: python_packages +2024-10-31 20:26:43,349 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:26:43,564 DEBUG SenderThread:10220 [sender.py:send():382] send: telemetry +2024-10-31 20:26:44,014 INFO wandb-upload_0:10220 [upload_job.py:push():131] Uploaded file /tmp/tmp6xpeoj7ewandb/5suw3rxm-wandb-metadata.json +2024-10-31 20:26:44,209 INFO Thread-12 :10220 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt +2024-10-31 20:26:44,209 INFO Thread-12 :10220 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:26:44,209 INFO Thread-12 :10220 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-metadata.json +2024-10-31 20:26:46,209 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:26:47,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:26:52,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:26:57,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:26:58,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:26:58,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:26:58,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:27:03,593 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:08,593 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:13,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:27:13,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:27:13,390 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:27:14,590 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:15,225 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml +2024-10-31 20:27:19,820 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:24,821 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:28,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:27:28,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:27:28,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:27:30,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:35,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:40,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:43,304 DEBUG SystemMonitor:10220 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-10-31 20:27:43,306 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:27:43,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:27:43,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:27:43,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:27:45,620 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:50,621 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:55,622 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:27:58,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:27:58,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:27:58,390 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:28:01,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:07,208 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:07,253 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:12,209 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:13,306 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:28:13,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:28:13,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:28:13,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:28:17,609 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:22,609 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:27,610 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:28,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:28:28,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:28:28,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:28:32,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:37,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:42,619 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:43,307 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:28:43,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:28:43,345 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:28:43,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:28:47,681 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:48,280 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:49,281 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:50,281 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:50,541 DEBUG SenderThread:10220 [sender.py:send():382] send: config +2024-10-31 20:28:50,541 DEBUG SenderThread:10220 [sender.py:send():382] send: config +2024-10-31 20:28:51,282 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:52,286 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:53,286 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:28:53,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:28:58,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:28:58,347 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:28:58,347 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:28:58,591 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:03,591 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:08,592 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:13,308 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:29:13,345 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:29:13,346 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:29:13,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:29:14,554 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:19,557 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:20,301 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml +2024-10-31 20:29:24,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:28,346 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:29:28,346 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:29:28,390 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:29:30,568 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:35,569 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:40,569 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:43,309 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:29:43,346 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:29:43,346 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:29:43,386 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:29:45,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:50,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:55,619 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:29:58,346 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:29:58,346 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:29:58,390 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:30:01,597 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:06,598 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:11,599 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:12,651 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:30:13,310 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:30:13,346 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:30:13,346 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:30:13,348 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:30:14,330 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:30:17,564 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:22,564 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:27,565 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:28,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:30:28,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:30:28,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:30:32,690 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:37,690 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:42,691 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:43,311 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:30:43,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:30:43,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:30:43,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:30:48,646 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:53,647 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:30:58,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:30:58,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:30:58,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:30:58,694 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:03,694 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:08,695 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:13,312 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:31:13,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:31:13,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:31:13,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:31:14,688 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:16,314 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:31:16,318 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:31:16,318 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:31:16,320 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:31:16,362 INFO Thread-12 :10220 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:31:18,363 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:31:20,321 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:25,321 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:28,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:31:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:31:28,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:31:30,730 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:35,730 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:40,731 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:43,313 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:31:43,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:31:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:31:43,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:31:46,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:51,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:56,681 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:31:58,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:31:58,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:31:58,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:32:02,637 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:07,637 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:12,638 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:13,314 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:32:13,444 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:32:13,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:32:13,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:32:17,691 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:19,708 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:32:19,710 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:32:19,711 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:32:19,712 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:32:20,396 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:32:22,397 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:32:22,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:27,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:32:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:32:28,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:32:33,632 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:38,632 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:43,332 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:32:43,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:32:43,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:32:43,445 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:32:43,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:48,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:53,681 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:32:58,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:32:58,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:32:58,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:32:59,639 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:04,639 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:09,640 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:13,316 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:33:13,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:33:13,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:33:13,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:33:14,658 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:19,659 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:23,214 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:33:23,217 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:33:23,217 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:33:23,219 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:33:23,429 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:33:24,429 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:33:25,220 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:33:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:33:28,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:33:30,655 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:35,655 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:40,656 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:43,317 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:33:43,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:33:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:33:43,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:33:45,684 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:50,685 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:55,686 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:33:58,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:33:58,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:33:58,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:34:01,639 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:06,640 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:11,640 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:13,318 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:34:13,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:34:13,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:34:13,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:34:16,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:21,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:26,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:27,188 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:34:27,190 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:34:27,191 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:34:27,195 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:34:27,462 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:34:28,445 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:34:28,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:34:28,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:34:28,463 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:34:32,692 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:37,693 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:42,693 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:43,318 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:34:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:34:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:34:43,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:34:48,674 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:53,674 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:34:58,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:34:58,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:34:58,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:34:59,624 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:04,625 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:09,626 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:13,319 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:35:13,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:35:13,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:35:13,446 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:35:14,664 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:19,665 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:24,665 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:28,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:35:28,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:35:28,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:35:30,663 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:30,739 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:35:30,742 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:35:30,742 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:35:30,743 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:35:31,497 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:35:32,498 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:35:35,744 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:40,745 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:43,320 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:35:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:35:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:35:43,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:35:46,681 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:51,682 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:56,683 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:35:58,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:35:58,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:35:58,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:36:01,698 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:06,698 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:11,699 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:13,321 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:36:13,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:36:13,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:36:13,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:36:16,723 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:21,724 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:26,724 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:28,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:36:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:36:28,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:36:32,691 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:34,354 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:36:34,356 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:36:34,357 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:36:34,358 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:36:34,531 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:36:36,532 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:36:38,359 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:43,322 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:36:43,446 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:36:43,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:36:43,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:36:43,629 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:48,630 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:53,630 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:36:58,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:36:58,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:36:58,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:36:58,641 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:03,642 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:08,642 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:13,323 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:37:13,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:37:13,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:37:13,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:37:14,637 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:19,637 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:24,638 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:37:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:37:28,447 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:37:30,634 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:35,635 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:38,085 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:37:38,087 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:37:38,087 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:37:38,089 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:37:38,591 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:37:40,592 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:37:41,090 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:43,326 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:37:43,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:37:43,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:37:43,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:37:46,715 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:51,716 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:56,716 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:37:58,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:37:58,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:37:58,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:38:02,684 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:07,684 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:12,685 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:13,327 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:38:13,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:38:13,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:38:13,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:38:18,652 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:23,652 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:38:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:38:28,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:38:29,629 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:34,630 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:39,630 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:41,312 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:38:41,314 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:38:41,314 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:38:41,316 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:38:41,625 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:38:42,625 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:38:43,328 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:38:43,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:38:43,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:38:43,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:38:44,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:49,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:54,723 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:38:58,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:38:58,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:38:58,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:39:00,666 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:05,667 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:10,668 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:13,329 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:39:13,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:39:13,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:39:13,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:39:15,724 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:20,724 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:25,725 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:28,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:39:28,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:39:28,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:39:31,701 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:36,701 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:41,702 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:43,330 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:39:43,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:39:43,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:39:43,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:39:44,594 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:39:44,596 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:39:44,596 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:39:44,598 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:39:44,660 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:39:46,661 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:39:47,599 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:52,599 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:57,600 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:39:58,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:39:58,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:39:58,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:40:02,679 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:07,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:12,680 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:13,331 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:40:13,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:40:13,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:40:13,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:40:17,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:22,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:27,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:28,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:40:28,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:40:28,449 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:40:33,629 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:38,630 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:43,335 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:40:43,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:40:43,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:40:43,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:40:43,787 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:48,116 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:40:48,118 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:40:48,118 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:40:48,119 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:40:48,695 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:40:49,159 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:49,696 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:40:54,159 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:40:58,529 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:40:58,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:40:58,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:40:59,758 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:04,758 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:09,759 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:13,333 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:41:13,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:41:13,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:41:13,578 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:41:15,740 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:20,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:25,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:28,529 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:41:28,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:41:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:41:31,715 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:36,716 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:41,716 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:43,335 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:41:43,529 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:41:43,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:41:43,570 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:41:46,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:51,457 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:41:51,459 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:41:51,459 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:41:51,460 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:41:51,731 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:41:52,499 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:53,732 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:41:57,499 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:41:58,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:41:58,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:41:58,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:42:02,812 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:07,813 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:12,813 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:13,335 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:42:13,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:42:13,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:42:13,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:42:18,776 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:23,777 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:28,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:42:28,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:42:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:42:29,720 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:34,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:39,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:43,336 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:42:43,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:42:43,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:42:43,578 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:42:44,726 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:49,726 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:54,727 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:42:54,786 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:42:54,790 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:42:54,790 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:42:54,791 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:42:55,766 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:42:57,767 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:42:58,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:42:58,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:42:58,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:42:59,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:04,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:09,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:13,337 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:43:13,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:43:13,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:43:13,578 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:43:15,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:20,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:25,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:28,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:43:28,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:43:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:43:30,725 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:35,726 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:40,727 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:43,338 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:43:43,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:43:43,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:43:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:43:45,802 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:50,803 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:55,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:43:58,142 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:43:58,144 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:43:58,144 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:43:58,145 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:43:58,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:43:58,530 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:43:58,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:43:58,801 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:43:59,802 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:44:01,803 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:06,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:11,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:13,338 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:44:13,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:44:13,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:44:13,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:44:17,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:22,748 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:27,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:28,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:44:28,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:44:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:44:32,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:37,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:42,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:43,339 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:44:43,530 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:44:43,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:44:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:44:48,768 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:53,768 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:44:58,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:44:58,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:44:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:44:59,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:01,691 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:45:01,716 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:45:01,716 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:45:01,717 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:45:01,836 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:45:03,837 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:45:05,735 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:10,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:13,340 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:45:13,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:45:13,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:45:13,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:45:15,816 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:20,817 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:25,817 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:28,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:45:28,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:45:28,578 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:45:31,704 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:36,704 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:41,705 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:43,341 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:45:43,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:45:43,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:45:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:45:46,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:51,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:56,743 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:45:58,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:45:58,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:45:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:46:02,730 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:05,122 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:46:05,123 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:46:05,124 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:46:05,125 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:46:05,885 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:46:07,886 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:46:08,163 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:13,163 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:13,342 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:46:13,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:46:13,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:46:13,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:46:18,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:23,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:28,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:46:28,531 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:46:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:46:29,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:34,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:39,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:43,343 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:46:43,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:46:43,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:46:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:46:44,795 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:49,795 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:54,796 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:46:58,531 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:46:58,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:46:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:47:00,735 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:05,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:08,412 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:47:08,414 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:47:08,415 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:47:08,416 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:47:08,919 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:47:09,920 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:47:11,455 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:13,344 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:47:13,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:47:13,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:47:13,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:47:16,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:21,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:26,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:28,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:47:28,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:47:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:47:31,778 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:36,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:41,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:43,344 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:47:43,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:47:43,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:47:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:47:46,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:51,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:56,783 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:47:58,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:47:58,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:47:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:48:01,823 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:06,823 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:11,525 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:48:11,526 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:48:11,527 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:48:11,528 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:48:11,957 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:48:12,570 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:13,345 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:48:13,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:48:13,532 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:48:13,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:48:13,959 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:48:17,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:22,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:27,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:28,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:48:28,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:48:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:48:32,771 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:37,771 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:42,772 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:43,346 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:48:43,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:48:43,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:48:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:48:48,731 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:53,731 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:48:58,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:48:58,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:48:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:48:58,794 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:03,795 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:08,795 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:13,347 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:49:13,532 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:49:13,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:49:13,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:49:14,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:15,246 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:49:15,248 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:49:15,248 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:49:15,249 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:49:15,992 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:49:17,993 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:49:20,291 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:25,292 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:28,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:49:28,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:49:28,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:49:30,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:35,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:40,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:43,348 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:49:43,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:49:43,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:49:43,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:49:45,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:50,751 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:55,751 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:49:58,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:49:58,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:49:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:50:00,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:05,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:10,771 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:13,349 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:50:13,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:50:13,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:50:13,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:50:16,761 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:18,961 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:50:18,962 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:50:18,962 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:50:18,964 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:50:19,030 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:50:22,003 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:22,032 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:50:27,003 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:28,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:50:28,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:50:28,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:50:32,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:37,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:42,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:43,350 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:50:43,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:50:43,533 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:50:43,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:50:47,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:52,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:57,783 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:50:58,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:50:58,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:50:58,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:51:02,789 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:07,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:12,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:13,351 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:51:13,533 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:51:13,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:51:13,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:51:18,786 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:22,090 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:51:22,092 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:51:22,092 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:51:22,093 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:51:23,065 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:51:24,065 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:51:24,094 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:28,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:51:28,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:51:28,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:51:29,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:34,755 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:39,755 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:43,352 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:51:43,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:51:43,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:51:43,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:51:44,760 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:49,760 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:54,761 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:51:58,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:51:58,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:51:58,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:51:59,806 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:04,806 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:09,807 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:13,353 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:52:13,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:52:13,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:52:13,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:52:15,806 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:20,807 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:25,456 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:52:25,459 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:52:25,459 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:52:25,460 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:52:26,098 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:52:26,461 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:28,099 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:52:28,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:52:28,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:52:28,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:52:31,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:36,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:41,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:43,354 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:52:43,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:52:43,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:52:43,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:52:46,740 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:51,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:56,741 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:52:58,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:52:58,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:52:58,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:53:01,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:06,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:11,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:13,357 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:53:13,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:53:13,534 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:53:13,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:53:17,773 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:22,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:27,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:28,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:53:28,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:53:28,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:53:29,206 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:53:29,208 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:53:29,209 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:53:29,214 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:53:30,139 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:53:32,140 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:53:33,214 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:38,215 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:43,216 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:43,358 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:53:43,534 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:53:43,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:53:43,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:53:48,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:53,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:53:58,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:53:58,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:53:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:53:59,732 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:04,732 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:09,733 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:13,359 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:54:13,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:54:13,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:54:13,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:54:14,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:19,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:24,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:28,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:54:28,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:54:28,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:54:29,794 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:32,845 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:54:32,847 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:54:32,847 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:54:32,849 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:54:33,174 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:54:34,174 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:54:34,849 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:39,850 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:43,359 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:54:43,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:54:43,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:54:43,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:54:45,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:50,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:55,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:54:58,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:54:58,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:54:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:55:00,753 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:05,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:10,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:13,360 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:55:13,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:55:13,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:55:13,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:55:15,816 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:20,817 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:25,817 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:28,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:55:28,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:55:28,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:55:31,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:36,126 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:55:36,128 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:55:36,129 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:55:36,130 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:55:36,208 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:55:37,131 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:38,209 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:55:42,131 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:43,363 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:55:43,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:55:43,535 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:55:43,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:55:47,735 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:52,735 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:57,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:55:58,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:55:58,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:55:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:56:02,805 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:07,805 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:12,806 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:13,362 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:56:13,535 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:56:13,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:56:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:56:18,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:23,791 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:28,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:56:28,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:56:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:56:29,745 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:34,746 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:39,475 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:56:39,477 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:56:39,477 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:56:39,480 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:56:40,243 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:56:40,480 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:42,244 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:56:43,363 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:56:43,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:56:43,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:56:43,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:56:45,768 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:50,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:55,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:56:58,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:56:58,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:56:58,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:57:01,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:06,748 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:11,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:13,365 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:57:13,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:57:13,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:57:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:57:17,710 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:22,710 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:27,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:28,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:57:28,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:57:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:57:32,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:37,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:42,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:42,892 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:57:42,894 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:57:42,894 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:57:42,896 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:57:43,278 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:57:43,364 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:57:43,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:57:43,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:57:43,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:57:44,278 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:57:47,789 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:52,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:57,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:57:58,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:57:58,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:57:58,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:58:03,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:08,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:13,365 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:58:13,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:58:13,536 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:58:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:58:13,797 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:18,797 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:23,798 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:28,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:58:28,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:58:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:58:29,781 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:34,781 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:39,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:43,366 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:58:43,536 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:58:43,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:58:43,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:58:45,734 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:46,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:58:46,584 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:58:46,585 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:58:46,586 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:58:47,313 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:58:48,313 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:58:51,587 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:56,588 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:58:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:58:58,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:58:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:59:01,853 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:06,854 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:11,854 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:13,367 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:59:13,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:59:13,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:59:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:59:17,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:22,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:27,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:28,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:59:28,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:59:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:59:32,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:37,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:42,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:43,368 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 20:59:43,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:59:43,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:59:43,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:59:48,787 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:49,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 20:59:49,722 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 20:59:49,724 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 20:59:49,725 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 20:59:50,351 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 20:59:52,352 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 20:59:54,726 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 20:59:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 20:59:58,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 20:59:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 20:59:59,781 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:04,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:09,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:13,369 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:00:13,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:00:13,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:00:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:00:15,778 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:20,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:25,779 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:28,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:00:28,537 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:00:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:00:30,785 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:35,786 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:40,786 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:43,370 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:00:43,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:00:43,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:00:43,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:00:46,761 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:51,761 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:53,139 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:00:53,141 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:00:53,142 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:00:53,144 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:00:53,386 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:00:54,387 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:00:57,144 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:00:58,537 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:00:58,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:00:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:01:02,746 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:07,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:12,748 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:13,371 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:01:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:01:13,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:01:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:01:18,725 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:23,726 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:01:28,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:01:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:01:28,745 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:33,745 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:38,746 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:43,372 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:01:43,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:01:43,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:01:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:01:43,764 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:48,765 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:53,765 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:01:56,334 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:01:56,336 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:01:56,336 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:01:56,338 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:01:56,422 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:01:58,424 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:01:58,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:01:58,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:01:58,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:01:58,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:03,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:08,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:13,373 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:02:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:02:13,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:02:13,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:02:14,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:19,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:24,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:02:28,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:02:28,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:02:30,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:35,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:40,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:43,374 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:02:43,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:02:43,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:02:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:02:45,758 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:50,759 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:55,759 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:02:58,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:02:58,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:02:58,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:02:59,723 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:02:59,724 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:02:59,725 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:02:59,726 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:03:00,457 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:03:01,727 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:02,459 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:03:06,728 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:11,728 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:13,375 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:03:13,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:03:13,538 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:03:13,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:03:16,792 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:21,793 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:26,793 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:28,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:03:28,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:03:28,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:03:32,710 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:37,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:42,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:43,378 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:03:43,538 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:03:43,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:03:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:03:47,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:52,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:57,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:03:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:03:58,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:03:58,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:04:02,807 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:03,354 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:04:03,357 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:04:03,357 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:04:03,359 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:04:03,493 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:04:04,493 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:04:08,359 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:13,360 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:13,376 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:04:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:04:13,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:04:13,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:04:18,809 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:23,810 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:04:28,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:04:28,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:04:29,758 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:34,759 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:39,759 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:43,378 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:04:43,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:04:43,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:04:43,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:04:44,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:49,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:54,791 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:04:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:04:58,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:04:58,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:05:00,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:05,723 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:06,839 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:05:06,842 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:05:06,842 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:05:06,844 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:05:07,545 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:05:08,545 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:05:10,844 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:13,378 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:05:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:05:13,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:05:13,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:05:16,799 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:21,800 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:26,801 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:05:28,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:05:28,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:05:32,765 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:37,766 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:42,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:43,379 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:05:43,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:05:43,539 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:05:43,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:05:47,798 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:52,798 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:57,799 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:05:58,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:05:58,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:05:58,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:06:02,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:07,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:10,096 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:06:10,098 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:06:10,099 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:06:10,100 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:06:10,580 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:06:12,581 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:06:13,101 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:13,380 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:06:13,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:06:13,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:06:13,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:06:18,752 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:23,753 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:28,539 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:06:28,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:06:28,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:06:29,712 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:34,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:39,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:43,381 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:06:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:06:43,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:06:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:06:44,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:49,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:54,771 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:06:58,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:06:58,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:06:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:06:59,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:04,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:09,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:13,321 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:07:13,323 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:07:13,324 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:07:13,325 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:07:13,382 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:07:13,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:07:13,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:07:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:07:13,673 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:07:14,674 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:07:14,819 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:19,819 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:24,820 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:28,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:07:28,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:07:28,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:07:30,719 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:35,720 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:40,720 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:43,383 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:07:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:07:43,540 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:07:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:07:45,821 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:50,822 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:55,822 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:07:58,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:07:58,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:07:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:08:01,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:06,739 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:11,739 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:13,384 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:08:13,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:08:13,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:08:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:08:16,813 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:17,113 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:08:17,115 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:08:17,115 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:08:17,116 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:08:17,709 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:08:18,709 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:08:22,117 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:27,118 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:28,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:08:28,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:08:28,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:08:32,776 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:37,777 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:42,777 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:43,384 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:08:43,540 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:08:43,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:08:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:08:48,772 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:53,773 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:08:58,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:08:58,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:08:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:08:59,751 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:04,751 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:09,752 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:13,385 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:09:13,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:09:13,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:09:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:09:14,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:19,811 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:20,491 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:09:20,493 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:09:20,494 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:09:20,495 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:09:20,778 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:09:22,779 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:09:25,496 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:28,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:09:28,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:09:28,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:09:30,773 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:35,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:40,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:43,386 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:09:43,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:09:43,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:09:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:09:46,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:51,738 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:56,739 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:09:58,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:09:58,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:09:58,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:10:01,789 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:06,789 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:11,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:13,387 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:10:13,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:10:13,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:10:13,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:10:17,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:22,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:24,069 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:10:24,072 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:10:24,072 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:10:24,073 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:10:24,815 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:10:26,816 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:10:28,074 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:28,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:10:28,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:10:28,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:10:33,753 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:38,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:43,388 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:10:43,541 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:10:43,541 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:10:43,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:10:44,746 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:49,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:54,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:10:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:10:58,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:10:58,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:11:00,721 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:05,722 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:10,723 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:13,389 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:11:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:11:13,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:11:13,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:11:15,735 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:20,736 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:25,737 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:27,747 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:11:27,751 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:11:27,752 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:11:27,753 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:11:27,852 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:11:28,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:11:28,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:11:28,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:11:28,853 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:11:30,749 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:35,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:40,750 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:43,390 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:11:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:11:43,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:11:43,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:11:46,729 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:51,730 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:56,731 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:11:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:11:58,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:11:58,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:12:01,742 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:06,743 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:11,743 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:13,391 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:12:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:12:13,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:12:13,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:12:16,825 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:21,825 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:26,826 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:28,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:12:28,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:12:28,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:12:31,331 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:12:31,334 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:12:31,334 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:12:31,335 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:12:31,887 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:12:32,336 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:32,887 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:12:37,336 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:42,337 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:43,392 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:12:43,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:12:43,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:12:43,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:12:47,814 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:52,814 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:57,815 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:12:58,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:12:58,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:12:58,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:13:03,717 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:08,717 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:13,392 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:13:13,542 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:13:13,542 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:13:13,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:13:13,752 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:18,753 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:23,753 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:28,543 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:13:28,543 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:13:28,544 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:13:28,835 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:33,835 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:35,075 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:13:35,078 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:13:35,078 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:13:35,079 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:13:35,931 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:13:36,932 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:13:39,080 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:43,393 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:13:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:13:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:13:43,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:13:44,766 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:49,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:54,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:13:58,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:13:58,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:13:58,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:13:59,799 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:04,800 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:09,800 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:13,394 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:14:13,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:14:13,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:14:13,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:14:14,842 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:19,842 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:24,843 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:14:28,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:14:28,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:14:29,845 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:34,846 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:38,265 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:14:38,267 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:14:38,267 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:14:38,268 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:14:38,966 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:14:38,966 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:14:40,307 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:43,395 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:14:43,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:14:43,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:14:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:14:45,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:50,769 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:55,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:14:58,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:14:58,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:14:58,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:15:01,755 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:06,756 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:11,756 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:13,396 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:15:13,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:15:13,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:15:13,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:15:17,754 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:22,755 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:27,755 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:28,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:15:28,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:15:28,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:15:32,777 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:37,778 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:42,259 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:15:42,261 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:15:42,261 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:15:42,262 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:15:43,004 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:15:43,004 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:15:43,303 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:43,397 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:15:43,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:15:43,573 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:15:43,575 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:15:48,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:53,768 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:15:58,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:15:58,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:15:58,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:15:58,853 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:03,853 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:08,854 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:13,398 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:16:13,573 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:16:13,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:16:13,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:16:14,787 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:19,788 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:24,788 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:16:28,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:16:28,614 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:16:30,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:35,770 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:40,771 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:43,399 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:16:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:16:43,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:16:43,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:16:45,439 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:16:45,441 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:16:45,441 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:16:45,445 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:16:46,037 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:16:46,483 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:47,037 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:16:51,483 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:56,484 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:16:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:16:58,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:16:58,575 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:17:01,820 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:06,820 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:11,821 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:13,399 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:17:13,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:17:13,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:17:13,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:17:17,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:22,780 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:27,781 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:28,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:17:28,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:17:28,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:17:33,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:38,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:43,400 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:17:43,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:17:43,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:17:43,618 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:17:43,819 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:48,819 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:49,140 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:17:49,143 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:17:49,144 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:17:49,147 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:17:50,071 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:17:51,071 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:17:54,183 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:17:58,574 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:17:58,574 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:17:58,576 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:17:59,787 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:04,787 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:09,788 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:13,401 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:18:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:18:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:18:13,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:18:14,828 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:19,828 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:24,829 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:28,581 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:18:28,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:18:28,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:18:30,828 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:35,828 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:40,829 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:43,402 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:18:43,581 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:18:43,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:18:43,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:18:46,825 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:51,826 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:52,267 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:18:52,269 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:18:52,269 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:18:52,270 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:18:53,142 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:18:53,142 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:18:57,272 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:18:58,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:18:58,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:18:58,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:19:02,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:07,790 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:12,791 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:13,403 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:19:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:19:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:19:13,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:19:18,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:23,767 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:28,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:19:28,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:19:28,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:19:29,764 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:34,765 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:39,765 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:43,404 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:19:43,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:19:43,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:19:43,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:19:44,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:49,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:54,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:19:55,607 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:19:55,609 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:19:55,610 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:19:55,634 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:19:56,180 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:19:57,181 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:19:58,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:19:58,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:19:58,582 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:19:59,871 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:04,871 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:09,872 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:13,405 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:20:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:20:13,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:20:13,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:20:15,806 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:20,807 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:25,807 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:28,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:20:28,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:20:28,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:20:31,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:36,774 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:41,775 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:43,406 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:20:43,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:20:43,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:20:43,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:20:46,814 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:51,815 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:56,815 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:20:58,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:20:58,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:20:58,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:20:58,985 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:20:58,986 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:20:58,987 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:20:58,989 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:20:59,215 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:21:01,216 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:21:01,989 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:06,990 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:11,990 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:13,407 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:21:13,582 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:21:13,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:21:13,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:21:17,803 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:22,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:27,804 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:28,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:21:28,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:21:28,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:21:32,855 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:37,856 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:42,856 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:43,408 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:21:43,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:21:43,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:21:43,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:21:48,781 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:53,782 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:21:58,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:21:58,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:21:58,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:21:58,850 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:02,402 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: partial_history +2024-10-31 21:22:02,403 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:22:02,404 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:22:02,406 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:22:03,251 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:03,251 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:22:04,734 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:09,734 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:12,257 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:13,408 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:22:13,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:22:13,583 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:22:13,583 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:22:14,808 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:20,261 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:20,595 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:25,596 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:28,783 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:22:28,832 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: stop_status +2024-10-31 21:22:28,832 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: stop_status +2024-10-31 21:22:31,051 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:34,270 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:36,739 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:37,669 DEBUG SenderThread:10220 [sender.py:send():382] send: exit +2024-10-31 21:22:37,670 INFO SenderThread:10220 [sender.py:send_exit():589] handling exit code: 0 +2024-10-31 21:22:37,670 INFO SenderThread:10220 [sender.py:send_exit():591] handling runtime: 3354 +2024-10-31 21:22:37,671 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:22:37,671 INFO SenderThread:10220 [sender.py:send_exit():597] send defer +2024-10-31 21:22:37,671 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,671 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 0 +2024-10-31 21:22:37,671 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,671 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-10-31 21:22:37,672 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 1 +2024-10-31 21:22:37,672 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,672 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 1 +2024-10-31 21:22:37,672 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,672 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-10-31 21:22:37,672 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 2 +2024-10-31 21:22:37,672 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,672 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 2 +2024-10-31 21:22:37,672 INFO HandlerThread:10220 [system_monitor.py:finish():203] Stopping system monitor +2024-10-31 21:22:37,672 DEBUG SystemMonitor:10220 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-10-31 21:22:37,672 INFO HandlerThread:10220 [interfaces.py:finish():202] Joined cpu monitor +2024-10-31 21:22:37,673 DEBUG SystemMonitor:10220 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-10-31 21:22:37,673 INFO HandlerThread:10220 [interfaces.py:finish():202] Joined disk monitor +2024-10-31 21:22:37,706 INFO HandlerThread:10220 [interfaces.py:finish():202] Joined gpu monitor +2024-10-31 21:22:37,707 INFO HandlerThread:10220 [interfaces.py:finish():202] Joined memory monitor +2024-10-31 21:22:37,707 INFO HandlerThread:10220 [interfaces.py:finish():202] Joined network monitor +2024-10-31 21:22:37,707 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,707 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-10-31 21:22:37,707 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 3 +2024-10-31 21:22:37,707 DEBUG SenderThread:10220 [sender.py:send():382] send: stats +2024-10-31 21:22:37,707 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,708 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 3 +2024-10-31 21:22:37,709 DEBUG SenderThread:10220 [sender.py:send():382] send: history +2024-10-31 21:22:37,709 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: summary_record +2024-10-31 21:22:37,710 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:22:37,711 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,711 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-10-31 21:22:37,711 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 4 +2024-10-31 21:22:37,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,711 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 4 +2024-10-31 21:22:37,711 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,711 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-10-31 21:22:37,711 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 5 +2024-10-31 21:22:37,711 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,711 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 5 +2024-10-31 21:22:37,712 DEBUG SenderThread:10220 [sender.py:send():382] send: summary +2024-10-31 21:22:37,713 INFO SenderThread:10220 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-10-31 21:22:37,713 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,713 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-10-31 21:22:37,713 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 6 +2024-10-31 21:22:37,713 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,713 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 6 +2024-10-31 21:22:37,713 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,713 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-10-31 21:22:37,714 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 7 +2024-10-31 21:22:37,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: status_report +2024-10-31 21:22:37,714 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:37,714 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 7 +2024-10-31 21:22:37,714 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:37,714 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-10-31 21:22:38,273 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:22:38,670 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: poll_exit +2024-10-31 21:22:39,688 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 8 +2024-10-31 21:22:39,688 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: poll_exit +2024-10-31 21:22:39,688 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:39,688 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 8 +2024-10-31 21:22:39,688 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:39,688 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-10-31 21:22:39,689 INFO SenderThread:10220 [job_builder.py:build():296] Attempting to build job artifact +2024-10-31 21:22:39,707 INFO SenderThread:10220 [job_builder.py:_get_source_type():426] is repo sourced job +2024-10-31 21:22:39,741 INFO SenderThread:10220 [job_builder.py:build():402] adding wandb-job metadata file +2024-10-31 21:22:39,760 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 9 +2024-10-31 21:22:39,761 DEBUG SenderThread:10220 [sender.py:send():382] send: artifact +2024-10-31 21:22:39,761 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:39,762 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 9 +2024-10-31 21:22:40,274 INFO Thread-12 :10220 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:40,670 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: poll_exit +2024-10-31 21:22:40,925 INFO wandb-upload_0:10220 [upload_job.py:push():86] Skipped uploading /singularity_home/.local/share/wandb/artifacts/staging/tmpryfn_3yz +2024-10-31 21:22:41,240 INFO wandb-upload_1:10220 [upload_job.py:push():89] Uploaded file /singularity_home/.local/share/wandb/artifacts/staging/tmpbt7q8231 +2024-10-31 21:22:42,617 INFO SenderThread:10220 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTMwMjQ5MDY3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQ4MzQ0MjQxNg==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTMwMjI2MzAyNA==', 'versionIndex': 0}}} +2024-10-31 21:22:42,617 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:42,617 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-10-31 21:22:42,618 INFO SenderThread:10220 [dir_watcher.py:finish():358] shutting down directory watcher +2024-10-31 21:22:43,275 INFO SenderThread:10220 [dir_watcher.py:finish():388] scan: /project/wandb/run-20241031_202642-fuo0pr3i/files +2024-10-31 21:22:43,276 INFO SenderThread:10220 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt requirements.txt +2024-10-31 21:22:43,276 INFO SenderThread:10220 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml config.yaml +2024-10-31 21:22:43,278 INFO SenderThread:10220 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-metadata.json wandb-metadata.json +2024-10-31 21:22:43,278 INFO SenderThread:10220 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json wandb-summary.json +2024-10-31 21:22:43,279 INFO SenderThread:10220 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log output.log +2024-10-31 21:22:43,281 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 10 +2024-10-31 21:22:43,281 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: poll_exit +2024-10-31 21:22:43,282 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:43,282 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 10 +2024-10-31 21:22:43,283 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:43,283 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-10-31 21:22:43,283 INFO SenderThread:10220 [file_pusher.py:finish():172] shutting down file pusher +2024-10-31 21:22:43,672 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: poll_exit +2024-10-31 21:22:43,672 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: poll_exit +2024-10-31 21:22:43,724 INFO wandb-upload_0:10220 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_202642-fuo0pr3i/files/requirements.txt +2024-10-31 21:22:43,744 INFO wandb-upload_2:10220 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_202642-fuo0pr3i/files/wandb-summary.json +2024-10-31 21:22:43,843 INFO wandb-upload_3:10220 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_202642-fuo0pr3i/files/output.log +2024-10-31 21:22:44,247 INFO wandb-upload_1:10220 [upload_job.py:push():131] Uploaded file /project/wandb/run-20241031_202642-fuo0pr3i/files/config.yaml +2024-10-31 21:22:44,447 INFO Thread-11 (_thread_body):10220 [sender.py:transition_state():617] send defer: 11 +2024-10-31 21:22:44,447 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:44,448 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 11 +2024-10-31 21:22:44,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:44,448 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 11 +2024-10-31 21:22:44,448 INFO SenderThread:10220 [file_pusher.py:join():178] waiting for file pusher +2024-10-31 21:22:44,448 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 12 +2024-10-31 21:22:44,448 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:44,448 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 12 +2024-10-31 21:22:44,448 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:44,448 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 12 +2024-10-31 21:22:44,448 INFO SenderThread:10220 [file_stream.py:finish():595] file stream finish called +2024-10-31 21:22:44,632 INFO SenderThread:10220 [file_stream.py:finish():599] file stream finish is done +2024-10-31 21:22:44,632 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 13 +2024-10-31 21:22:44,632 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:44,633 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 13 +2024-10-31 21:22:44,633 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:44,633 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 13 +2024-10-31 21:22:44,633 INFO SenderThread:10220 [sender.py:transition_state():617] send defer: 14 +2024-10-31 21:22:44,633 DEBUG SenderThread:10220 [sender.py:send():382] send: final +2024-10-31 21:22:44,633 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: defer +2024-10-31 21:22:44,633 DEBUG SenderThread:10220 [sender.py:send():382] send: footer +2024-10-31 21:22:44,633 INFO HandlerThread:10220 [handler.py:handle_request_defer():172] handle defer: 14 +2024-10-31 21:22:44,634 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: defer +2024-10-31 21:22:44,634 INFO SenderThread:10220 [sender.py:send_request_defer():613] handle sender defer: 14 +2024-10-31 21:22:44,634 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: poll_exit +2024-10-31 21:22:44,634 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: poll_exit +2024-10-31 21:22:44,635 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: poll_exit +2024-10-31 21:22:44,635 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: poll_exit +2024-10-31 21:22:44,635 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: server_info +2024-10-31 21:22:44,635 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: server_info +2024-10-31 21:22:44,635 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: get_summary +2024-10-31 21:22:44,637 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: sampled_history +2024-10-31 21:22:44,640 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: internal_messages +2024-10-31 21:22:44,640 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: job_info +2024-10-31 21:22:44,818 DEBUG SenderThread:10220 [sender.py:send_request():409] send_request: job_info +2024-10-31 21:22:44,818 INFO MainThread:10220 [wandb_run.py:_footer_history_summary_info():3866] rendering history +2024-10-31 21:22:44,819 INFO MainThread:10220 [wandb_run.py:_footer_history_summary_info():3898] rendering summary +2024-10-31 21:22:44,820 INFO MainThread:10220 [wandb_run.py:_footer_sync_info():3825] logging synced files +2024-10-31 21:22:44,820 DEBUG HandlerThread:10220 [handler.py:handle_request():146] handle_request: shutdown +2024-10-31 21:22:44,820 INFO HandlerThread:10220 [handler.py:finish():869] shutting down handler +2024-10-31 21:22:45,641 INFO WriterThread:10220 [datastore.py:close():296] close: /project/wandb/run-20241031_202642-fuo0pr3i/run-fuo0pr3i.wandb +2024-10-31 21:22:45,818 INFO SenderThread:10220 [sender.py:finish():1572] shutting down sender +2024-10-31 21:22:45,818 INFO SenderThread:10220 [file_pusher.py:finish():172] shutting down file pusher +2024-10-31 21:22:45,818 INFO SenderThread:10220 [file_pusher.py:join():178] waiting for file pusher diff --git a/wandb/run-20241031_202642-fuo0pr3i/logs/debug.log b/wandb/run-20241031_202642-fuo0pr3i/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f2e3bd50ba5b430058055cfb0e15f25f7790c2d8 --- /dev/null +++ b/wandb/run-20241031_202642-fuo0pr3i/logs/debug.log @@ -0,0 +1,30 @@ +2024-10-31 20:26:42,661 INFO MainThread:10149 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Configure stats pid to 10149 +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'CPT en-ja'} +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20241031_202642-fuo0pr3i/logs/debug.log +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20241031_202642-fuo0pr3i/logs/debug-internal.log +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:init():566] calling init triggers +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'NO_SHARD', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['183190553', '/project/trans/datasets/train/documents_text_document'], 'valid_data_path': ['61085876', '/project/trans/datasets/valid/documents_text_document'], 'test_data_path': ['61085876', '/project/trans/datasets/valid/documents_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'llm-jp-v3-3.7b_ja-en_3M-pairs_train_2024-10-31-20:26:21', 'wandb_project': 'trans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', 'save': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_3M-pairs', 'base_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b-instruct', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 200, 'save_interval': 200, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 0.00015, 'lr_decay_style': 'cosine', 'lr_decay_iters': 50, 'lr_warmup_iters': 5, 'min_lr': 1.5e-05, 'train_iters': 50, 'train_samples': None, 'global_batch_size': 256, 'micro_batch_size': 1, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 10, 'hf_repo_id': 'koichi12/llm-jp-v3-3.7b_ja-en_3M-pairs', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 1, 'padded_vocab_size': 99584, 'gradient_accumulation_steps': 256} +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:init():616] starting backend +2024-10-31 20:26:42,662 INFO MainThread:10149 [wandb_init.py:init():620] setting up manager +2024-10-31 20:26:42,668 INFO MainThread:10149 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-10-31 20:26:42,668 INFO MainThread:10149 [wandb_init.py:init():628] backend started and connected +2024-10-31 20:26:42,673 INFO MainThread:10149 [wandb_init.py:init():720] updated telemetry +2024-10-31 20:26:42,772 INFO MainThread:10149 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-10-31 20:26:43,211 INFO MainThread:10149 [wandb_run.py:_on_init():2262] communicating current version +2024-10-31 20:26:43,246 INFO MainThread:10149 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-10-31 20:26:43,246 INFO MainThread:10149 [wandb_init.py:init():804] starting run threads in backend +2024-10-31 20:26:43,345 INFO MainThread:10149 [wandb_run.py:_console_start():2241] atexit reg +2024-10-31 20:26:43,345 INFO MainThread:10149 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-10-31 20:26:43,345 INFO MainThread:10149 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-10-31 20:26:43,345 INFO MainThread:10149 [wandb_run.py:_redirect():2186] Redirects installed. +2024-10-31 20:26:43,346 INFO MainThread:10149 [wandb_init.py:init():847] run started, returning control to user process +2024-10-31 20:28:50,540 INFO MainThread:10149 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'LlamaForCausalLM', 'activation_function': 'silu', 'hidden_size': 3072, 'model_type': 'llama', 'max_position_embeddings': 4096, 'num_attention_heads': 24, 'num_hidden_layers': 28} +2024-10-31 20:28:50,540 INFO MainThread:10149 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 1} +2024-10-31 21:22:45,821 WARNING MsgRouterThr:10149 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/run-20241031_202642-fuo0pr3i/run-fuo0pr3i.wandb b/wandb/run-20241031_202642-fuo0pr3i/run-fuo0pr3i.wandb new file mode 100644 index 0000000000000000000000000000000000000000..07eadbb1832c342734e83092a030d037e6b30b79 Binary files /dev/null and b/wandb/run-20241031_202642-fuo0pr3i/run-fuo0pr3i.wandb differ diff --git a/wandb/run-20241101_034513-twy91j38/files/config.yaml b/wandb/run-20241101_034513-twy91j38/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38f85cda4b2d43b69291d482e99e8ed35e00f865 --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/files/config.yaml @@ -0,0 +1,340 @@ +wandb_version: 1 + +sharding_strategy: + desc: null + value: SHARD_GRAD_OP +checkpoint_type: + desc: null + value: LOCAL_STATE_DICT +fsdp_activation_checkpointing: + desc: null + value: true +fsdp_cpu_offload: + desc: null + value: false +low_cpu_fsdp: + desc: null + value: false +no_meta_device: + desc: null + value: false +data_path: + desc: null + value: null +split: + desc: null + value: 969, 30, 1 +train_data_path: + desc: null + value: + - '183224836' + - /project/trans/datasets/processed_ja-en/train/documents_text_document +valid_data_path: + desc: null + value: + - '61084836' + - /project/trans/datasets/processed_ja-en/valid/documents_text_document +test_data_path: + desc: null + value: + - '61084836' + - /project/trans/datasets/processed_ja-en/valid/documents_text_document +data_cache_path: + desc: null + value: null +vocab_size: + desc: null + value: null +vocab_file: + desc: null + value: null +merge_file: + desc: null + value: null +seq_length: + desc: null + value: 1024 +num_workers: + desc: null + value: 4 +tokenizer_type: + desc: null + value: HFPreTrainedTokenizer +tokenizer_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b +reset_position_ids: + desc: null + value: false +reset_attention_mask: + desc: null + value: false +eod_mask_loss: + desc: null + value: false +retro_return_doc_ids: + desc: null + value: false +short_seq_prob: + desc: null + value: 0.1 +vocab_extra_ids: + desc: null + value: 0 +seed: + desc: null + value: 1234 +use_mpi: + desc: null + value: false +wandb_entity: + desc: null + value: iwakawa-koichi-q5-tohoku-nlp6723 +wandb_name: + desc: null + value: llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:44:59 +wandb_project: + desc: null + value: trans_experiment +quantization: + desc: null + value: false +use_freeze_layers: + desc: null + value: false +freeze_layers: + desc: null + value: null +bf16: + desc: null + value: true +fp16: + desc: null + value: false +mixed_precision: + desc: null + value: true +param_dtype: + desc: null + value: null +load: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs +save: + desc: null + value: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs +base_model: + desc: null + value: /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b +use_better_transformer: + desc: null + value: false +grad_clip_norm: + desc: null + value: 1.0 +eval_interval: + desc: null + value: 20 +save_interval: + desc: null + value: 50 +eval_iters: + desc: null + value: 10 +optimizer: + desc: null + value: anyprecision +lr: + desc: null + value: 0.00015 +lr_decay_style: + desc: null + value: cosine +lr_decay_iters: + desc: null + value: 698 +lr_warmup_iters: + desc: null + value: 70 +min_lr: + desc: null + value: 1.5e-05 +train_iters: + desc: null + value: 698 +train_samples: + desc: null + value: null +global_batch_size: + desc: null + value: 256 +micro_batch_size: + desc: null + value: 12 +make_vocab_size_divisible_by: + desc: null + value: 128 +sliding_window_size: + desc: null + value: 131072 +skip_batch: + desc: null + value: null +no_save_optimizer_state: + desc: null + value: false +continual_pretraining: + desc: null + value: false +instruction_tuning: + desc: null + value: false +direct_preference_optimization: + desc: null + value: false +attention_dropout: + desc: null + value: 0.1 +hidden_dropout: + desc: null + value: 0.1 +weight_decay: + desc: null + value: 0.1 +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.95 +adam_eps: + desc: null + value: 1.0e-08 +hf_transformer_model_dir: + desc: null + value: null +instruction_train_data_path: + desc: null + value: null +instruction_valid_data_path: + desc: null + value: null +epoch: + desc: null + value: null +instruction_dataset_size: + desc: null + value: null +save_sampler_state: + desc: null + value: false +label_smoothing: + desc: null + value: 0.0 +save_n_checkpoints: + desc: null + value: 3 +hf_repo_id: + desc: null + value: koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs +create_public_hf_repo: + desc: null + value: false +upload_all_checkpoints_to_hf: + desc: null + value: true +hf_upload_retry_limit: + desc: null + value: 2 +exit_duration_in_mins: + desc: null + value: null +source_key: + desc: null + value: null +target_key: + desc: null + value: null +attn_implementation: + desc: null + value: flash_attention_2 +efficient_instruction_tuning: + desc: null + value: false +remove_padding_masking: + desc: null + value: false +save_start_iter: + desc: null + value: null +valid_micro_batch_size: + desc: null + value: 1 +rank: + desc: null + value: 0 +world_size: + desc: null + value: 2 +padded_vocab_size: + desc: null + value: 99584 +gradient_accumulation_steps: + desc: null + value: 10 +_wandb: + desc: null + value: + python_version: 3.10.12 + cli_version: 0.16.3 + framework: huggingface + huggingface_version: 4.43.3 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1730400313.590059 + t: + 1: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 2: + - 1 + - 11 + - 49 + - 55 + - 71 + - 105 + 3: + - 13 + - 16 + - 23 + 4: 3.10.12 + 5: 0.16.3 + 6: 4.43.3 + 8: + - 5 + 13: linux-x86_64 +model_architecture: + desc: null + value: LlamaForCausalLM +activation_function: + desc: null + value: silu +hidden_size: + desc: null + value: 3072 +model_type: + desc: null + value: llama +max_position_embeddings: + desc: null + value: 4096 +num_attention_heads: + desc: null + value: 24 +num_hidden_layers: + desc: null + value: 28 diff --git a/wandb/run-20241101_034513-twy91j38/files/output.log b/wandb/run-20241101_034513-twy91j38/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..621a19207c907dc4d522cc4657c13151342f2e13 --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/files/output.log @@ -0,0 +1,126 @@ +Created Hugging Face repository with ID koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs. +Clearing GPU cache for all ranks +--> Running with torch torch_distributed debug set to detail +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +You are attempting to use Flash Attention 2.0 without specifying a torch dtype. This might lead to unexpected behaviour +You are attempting to use Flash Attention 2.0 with a model not initialized on GPU. Make sure to move the model to GPU after initializing it on CPU with `model.to('cuda')`. +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaForCausalLM is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` +Flash Attention 2.0 only supports torch.float16 and torch.bfloat16 dtypes, but the current dype in LlamaModel is torch.float32. You should run training or inference using Automatic Mixed-Precision via the `with torch.autocast(device_type='torch_device'):` decorator, or load the model with the `torch_dtype` argument. Example: `model = AutoModel.from_pretrained("openai/whisper-tiny", attn_implementation="flash_attention_2", torch_dtype=torch.float16)` + +Loading checkpoint shards: 50%|█████ | 1/2 [00:05<00:05, 5.13s/it] +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs, skipping model loading +--> Model /share/pretrained_lm/llm-jp/llm-jp-v3-3.7b + +Loading checkpoint shards: 100%|██████████| 2/2 [00:07<00:00, 3.88s/it] +BFloat16 enabled for mixed precision - using bfSixteen policy +Let split = None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +Building a BlendedDataset for a single MegatronDataset +Unable to save the indexes because path_to_cache is None +--> applying fsdp activation checkpointing... + > datasets target sizes (minimum size): + train: 178688 + validation: 89600 + test: 2560 +> building train, validation, and test datasets for GPT ... +> finished creating GPT datasets ... +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +No checkpoint found in /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs, skipping optimizer loading +File not found: /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +Unable to read latest iteration from /work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs/latest_iteration.txt +model info: FullyShardedDataParallel( + (_fsdp_wrapped_module): LlamaForCausalLM( + (model): LlamaModel( + (embed_tokens): Embedding(99584, 3072) + (layers): ModuleList( + (0-27): 28 x FullyShardedDataParallel( + (_fsdp_wrapped_module): CheckpointWrapper( + (_checkpoint_wrapped_module): LlamaDecoderLayer( + (self_attn): LlamaFlashAttention2( + (q_proj): Linear(in_features=3072, out_features=3072, bias=False) + (k_proj): Linear(in_features=3072, out_features=3072, bias=False) + (v_proj): Linear(in_features=3072, out_features=3072, bias=False) + (o_proj): Linear(in_features=3072, out_features=3072, bias=False) + (rotary_emb): LlamaRotaryEmbedding() + ) + (mlp): LlamaMLP( + (gate_proj): Linear(in_features=3072, out_features=8192, bias=False) + (up_proj): Linear(in_features=3072, out_features=8192, bias=False) + (down_proj): Linear(in_features=8192, out_features=3072, bias=False) + (act_fn): SiLU() + ) + (input_layernorm): LlamaRMSNorm() + (post_attention_layernorm): LlamaRMSNorm() + ) + ) + ) + ) + (norm): LlamaRMSNorm() + (rotary_emb): LlamaRotaryEmbedding() + ) + (lm_head): Linear(in_features=3072, out_features=99584, bias=False) + ) +) +model config: LlamaConfig { + "_name_or_path": "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "architectures": [ + "LlamaForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "eos_token_id": 2, + "hidden_act": "silu", + "hidden_size": 3072, + "initializer_range": 0.02, + "intermediate_size": 8192, + "label_smoothing": 0.0, + "max_position_embeddings": 4096, + "mlp_bias": false, + "model_type": "llama", + "num_attention_heads": 24, + "num_hidden_layers": 28, + "num_key_value_heads": 24, + "pretraining_tp": 1, + "rms_norm_eps": 1e-05, + "rope_scaling": null, + "rope_theta": 10000, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "4.43.3", + "use_cache": false, + "vocab_size": 99584 +} +------------------------------------------------------------------ +iteration: 1 , TFLOPS: 114.59595714447318, Tokens per sec: 10456.36403310636, Loss: 3.0579614639282227 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 2 , TFLOPS: 119.17856755933255, Tokens per sec: 10874.506556749353, Loss: 2.916849136352539 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 3 , TFLOPS: 122.22886774197347, Tokens per sec: 11152.832685477732, Loss: 2.726020336151123 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 4 , TFLOPS: 122.20061461764121, Tokens per sec: 11150.254715360354, Loss: 2.6824560165405273 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 5 , TFLOPS: 122.24375667623931, Tokens per sec: 11154.191233550699, Loss: 2.622641086578369 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 6 , TFLOPS: 122.2060860770871, Tokens per sec: 11150.753961347595, Loss: 2.6402499675750732 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 7 , TFLOPS: 122.22669120774827, Tokens per sec: 11152.634086550193, Loss: 2.6120753288269043 +------------------------------------------------------------------ +------------------------------------------------------------------ +iteration: 8 , TFLOPS: 122.33492004287078, Tokens per sec: 11162.509479427214, Loss: 2.5807323455810547 +------------------------------------------------------------------ \ No newline at end of file diff --git a/wandb/run-20241101_034513-twy91j38/files/requirements.txt b/wandb/run-20241101_034513-twy91j38/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..34a2774e444bdc395654ccf8ce6da6833c7bc1ee --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/files/requirements.txt @@ -0,0 +1,375 @@ +absl-py==2.1.0 +accelerate==0.23.0 +aiohttp==3.9.1 +aiosignal==1.3.1 +annotated-types==0.6.0 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +apex==0.1 +appdirs==1.4.4 +argon2-cffi-bindings==21.2.0 +argon2-cffi==23.1.0 +astroid==3.2.4 +asttokens==2.4.1 +astunparse==1.6.3 +async-timeout==4.0.3 +attrs==23.2.0 +audioread==3.0.1 +beautifulsoup4==4.12.3 +bert-score==0.3.13 +bleach==6.1.0 +blis==0.7.11 +build==1.2.1 +cachecontrol==0.14.0 +cachetools==5.3.2 +catalogue==2.0.10 +certifi==2024.2.2 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +cleo==2.1.0 +click==8.1.7 +cloudpathlib==0.16.0 +cloudpickle==3.0.0 +cmake==3.28.1 +colorama==0.4.6 +comm==0.2.1 +confection==0.1.4 +contourpy==1.2.0 +cramjam==2.8.3 +crashtest==0.4.1 +cryptography==43.0.0 +cubinlinker==0.3.0+2.g405ac64 +cuda-python==12.3.0rc4+9.gdb8c48a.dirty +cudf==23.12.0 +cugraph-dgl==23.12.0 +cugraph-service-client==23.12.0 +cugraph-service-server==23.12.0 +cugraph==23.12.0 +cuml==23.12.0 +cupy-cuda12x==12.3.0 +cycler==0.12.1 +cymem==2.0.8 +cython==3.0.8 +dask-cuda==23.12.0 +dask-cudf==23.12.0 +dask==2023.11.0 +dataclasses-json==0.6.7 +dataproperty==1.0.1 +datasets==2.20.0 +debugpy==1.8.1 +decorator==5.1.1 +defusedxml==0.7.1 +dill==0.3.8 +distlib==0.3.8 +distributed==2023.11.0 +distro==1.9.0 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +dulwich==0.21.7 +einops==0.7.0 +emoji==2.12.1 +entmax==1.3 +evaluate==0.4.2 +exceptiongroup==1.2.0 +execnet==2.0.2 +executing==2.0.1 +expecttest==0.1.3 +fastjsonschema==2.19.1 +fastparquet==2023.10.1 +fastrlock==0.8.2 +filelock==3.13.1 +flash-attn==2.4.2 +fonttools==4.48.1 +frozenlist==1.4.1 +fsspec==2023.12.2 +fugashi==1.3.2 +fuzzywuzzy==0.18.0 +gast==0.5.4 +gitdb==4.0.11 +gitpython==3.1.43 +google-auth-oauthlib==0.4.6 +google-auth==2.27.0 +graphsurgeon==0.4.6 +greenlet==3.0.3 +grpcio==1.60.1 +h11==0.14.0 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.24.5 +hydra-core==1.3.2 +hypothesis==5.35.1 +idna==3.6 +importlib-metadata==7.0.1 +iniconfig==2.0.0 +installer==0.7.0 +intel-openmp==2021.4.0 +ipadic==1.0.0 +ipykernel==6.29.2 +ipython-genutils==0.2.0 +ipython==8.21.0 +isort==5.13.2 +jaraco.classes==3.4.0 +jedi==0.19.1 +jeepney==0.8.0 +jinja2==3.1.3 +jiter==0.5.0 +joblib==1.3.2 +json5==0.9.14 +jsonargparse==3.13.1 +jsonlines==4.0.0 +jsonnet==0.19.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema-specifications==2023.12.1 +jsonschema==4.21.1 +jupyter-client==8.6.0 +jupyter-core==5.7.1 +jupyter-tensorboard==0.2.0 +jupyterlab-pygments==0.3.0 +jupyterlab-server==1.2.0 +jupyterlab==2.3.2 +jupytext==1.16.1 +keyring==24.3.1 +kiwisolver==1.4.5 +langchain-community==0.2.12 +langchain-core==0.2.31 +langchain-huggingface==0.0.2 +langchain-openai==0.1.21 +langchain-text-splitters==0.2.2 +langchain==0.2.13 +langcodes==3.3.0 +langsmith==0.1.99 +lazy-loader==0.3 +levenshtein==0.25.1 +librosa==0.10.1 +lightning-utilities==0.11.6 +llm-jp-eval==1.4.0 +llvmlite==0.40.1 +lm-eval==0.3.0 +locket==1.0.0 +logzero==1.7.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +markdown==3.5.2 +markupsafe==2.1.4 +marshmallow==3.21.3 +matplotlib-inline==0.1.6 +matplotlib==3.8.2 +mbstrdecoder==1.1.3 +mccabe==0.7.0 +mdit-py-plugins==0.4.0 +mdurl==0.1.2 +mecab-python3==1.0.6 +mistune==3.0.2 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +mkl==2021.1.1 +mock==5.1.0 +mojimoji==0.0.13 +more-itertools==9.1.0 +mpmath==1.3.0 +msgpack==1.0.7 +multidict==6.0.4 +multiprocess==0.70.16 +murmurhash==1.0.10 +mypy-extensions==1.0.0 +nbclient==0.9.0 +nbconvert==7.16.0 +nbformat==5.9.2 +neologdn==0.5.3 +nest-asyncio==1.6.0 +networkx==2.6.3 +ninja==1.11.1.1 +nltk==3.8.1 +notebook==6.4.10 +numba==0.57.1+1.g1ff679645 +numexpr==2.10.1 +numpy==1.24.4 +nvfuser==0.1.4a0+d0bb811 +nvidia-dali-cuda120==1.34.0 +nvidia-pyindex==1.0.9 +nvtx==0.2.5 +oauthlib==3.2.2 +omegaconf==2.3.0 +onnx==1.15.0rc2 +openai==1.40.6 +opencv==4.7.0 +optree==0.10.0 +orjson==3.10.7 +packaging==23.2 +pandas==2.2.2 +pandocfilters==1.5.1 +parso==0.8.3 +partd==1.4.1 +pathvalidate==3.2.0 +peft==0.5.0 +pexpect==4.9.0 +pillow==10.2.0 +pip==24.0 +pkginfo==1.11.1 +plac==1.4.3 +platformdirs==4.2.0 +pluggy==1.4.0 +ply==3.11 +poetry-core==1.9.0 +poetry-plugin-export==1.8.0 +poetry==1.8.3 +polygraphy==0.49.4 +pooch==1.8.0 +portalocker==2.10.1 +preshed==3.0.9 +prettytable==3.9.0 +prometheus-client==0.19.0 +prompt-toolkit==3.0.43 +protobuf==4.24.4 +psutil==5.9.4 +ptxcompiler==0.8.1+2.g0d406d6 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow-hotfix==0.6 +pyarrow==15.0.2 +pyasn1-modules==0.3.0 +pyasn1==0.5.1 +pybind11-global==2.11.1 +pybind11==2.11.1 +pycocotools==2.0+nv0.8.0 +pycountry==24.6.1 +pycparser==2.21 +pydantic-core==2.16.2 +pydantic==2.6.1 +pygments==2.17.2 +pylibcugraph==23.12.0 +pylibcugraphops==23.12.0 +pylibraft==23.12.0 +pylint==3.2.6 +pynvml==11.4.1 +pyparsing==3.1.1 +pyproject-hooks==1.1.0 +pytablewriter==1.2.0 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==13.0 +pytest-shard==0.1.2 +pytest-xdist==3.5.0 +pytest==8.0.0 +python-dateutil==2.8.2 +python-dotenv==1.0.0 +python-hostlist==1.23.0 +python-levenshtein==0.25.1 +pytorch-lightning==2.4.0 +pytorch-quantization==2.1.2 +pytz==2023.3.post1 +pyyaml==6.0.1 +pyzmq==25.1.2 +raft-dask==23.12.0 +rapidfuzz==3.9.6 +rapids-dask-dependency==23.12.1 +referencing==0.33.0 +regex==2023.12.25 +requests-oauthlib==1.3.1 +requests-toolbelt==1.0.0 +requests==2.32.3 +rhoknp==1.7.0 +rich==13.7.0 +rmm==23.12.0 +rouge-score==0.1.2 +rpds-py==0.17.1 +rsa==4.9 +sacrebleu==2.4.2 +safetensors==0.4.3 +scikit-learn==1.5.1 +scipy==1.12.0 +secretstorage==3.3.3 +send2trash==1.8.2 +sentence-transformers==3.0.1 +sentencepiece==0.1.99 +sentry-sdk==2.12.0 +setproctitle==1.3.3 +setuptools==68.2.2 +shellingham==1.5.4 +six==1.16.0 +smart-open==6.4.0 +smmap==5.0.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.12.1 +soupsieve==2.5 +soxr==0.3.7 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +spacy==3.7.2 +sphinx-glpi-theme==0.6 +sqlalchemy==2.0.32 +sqlitedict==2.1.0 +srsly==2.4.8 +stack-data==0.6.3 +sumeval==0.2.2 +sympy==1.12 +tabledata==1.3.3 +tabulate==0.9.0 +tbb==2021.11.0 +tblib==3.0.0 +tcolorpy==0.1.6 +tenacity==8.5.0 +tensorboard-data-server==0.6.1 +tensorboard-plugin-wit==1.8.1 +tensorboard==2.9.0 +tensorrt==8.6.3 +terminado==0.18.0 +termplotlib==0.3.9 +text-generation==0.7.0 +thinc==8.2.3 +threadpoolctl==3.2.0 +thriftpy2==0.4.17 +tiktoken==0.7.0 +tinycss2==1.2.1 +tokenizers==0.19.1 +toml==0.10.2 +tomli==2.0.1 +tomlkit==0.13.2 +toolz==0.12.1 +torch-tensorrt==2.3.0a0 +torch==2.3.0a0+ebedce2 +torchdata==0.7.1a0 +torchmetrics==0.10.3 +torchtext==0.17.0a0 +torchvision==0.18.0a0 +tornado==6.4 +tqdm-multiprocess==0.0.11 +tqdm==4.66.5 +traitlets==5.9.0 +transformer-engine==1.3.0+5b90b7f +transformers==4.43.3 +treelite-runtime==3.9.1 +treelite==3.9.1 +triton==2.2.0+e28a256 +trove-classifiers==2024.7.2 +typepy==1.3.2 +typer==0.9.0 +types-dataclasses==0.6.6 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.1 +ucx-py==0.35.0 +uff==0.6.9 +ujson==5.8.0 +unbabel-comet==2.2.2 +unidic-lite==1.0.8 +urllib3==1.26.18 +virtualenv==20.26.3 +wandb==0.16.3 +wasabi==1.1.2 +wcwidth==0.2.13 +weasel==0.3.4 +webencodings==0.5.1 +werkzeug==3.0.1 +wheel==0.42.0 +word2number==1.1 +xdoctest==1.0.2 +xgboost==1.7.6 +xmltodict==0.13.0 +xxhash==3.4.1 +yarl==1.9.4 +zict==3.0.0 +zipp==3.17.0 +zstandard==0.23.0 \ No newline at end of file diff --git a/wandb/run-20241101_034513-twy91j38/files/wandb-metadata.json b/wandb/run-20241101_034513-twy91j38/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..495ed989ec5b9fd029f64511b0bc42197bebccb3 --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/files/wandb-metadata.json @@ -0,0 +1,314 @@ +{ + "os": "Linux-5.15.0-91-generic-x86_64-with-glibc2.35", + "python": "3.10.12", + "heartbeatAt": "2024-10-31T18:45:14.181243", + "startedAt": "2024-10-31T18:45:13.576906", + "docker": null, + "cuda": null, + "args": [ + "--seq-length", + "1024", + "--sliding-window-size", + "131072", + "--micro-batch-size", + "12", + "--valid_micro_batch_size", + "1", + "--global-batch-size", + "256", + "--train-iters", + "698", + "--tokenizer-type", + "HFPreTrainedTokenizer", + "--tokenizer-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--train-data-path", + "183224836", + "/project/trans/datasets/processed_ja-en/train/documents_text_document", + "--valid-data-path", + "61084836", + "/project/trans/datasets/processed_ja-en/valid/documents_text_document", + "--test-data-path", + "61084836", + "/project/trans/datasets/processed_ja-en/valid/documents_text_document", + "--lr", + "1.5e-4", + "--min-lr", + "1.5e-5", + "--lr-decay-style", + "cosine", + "--lr-warmup-iters", + "70", + "--lr-decay-iters", + "698", + "--weight-decay", + "0.1", + "--grad-clip-norm", + "1.0", + "--optimizer", + "anyprecision", + "--adam-beta1", + "0.9", + "--adam-beta2", + "0.95", + "--adam-eps", + "1e-8", + "--save-interval", + "50", + "--eval-interval", + "20", + "--eval-iters", + "10", + "--bf16", + "--mixed-precision", + "--base-model", + "/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b", + "--save", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--load", + "/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--num-workers", + "4", + "--fsdp-activation-checkpointing", + "--sharding-strategy", + "SHARD_GRAD_OP", + "--checkpoint-type", + "LOCAL_STATE_DICT", + "--save-n-checkpoints", + "3", + "--upload-all-checkpoints-to-hf", + "--hf-upload-retry-limit", + "2", + "--hf-repo-id", + "koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs", + "--wandb-entity", + "iwakawa-koichi-q5-tohoku-nlp6723", + "--wandb-project", + "trans_experiment", + "--wandb-name", + "llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:44:59" + ], + "state": "running", + "program": "/project/examples/finetuning.py", + "codePathLocal": "examples/finetuning.py", + "codePath": "examples/finetuning.py", + "git": { + "remote": "https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git", + "commit": "3b2976faebe2228c39adb20194a29b785a37defe" + }, + "email": null, + "root": "/project", + "host": "gpu-koiwa-00", + "username": "koiwa", + "executable": "/usr/bin/python", + "cpu_count": 36, + "cpu_count_logical": 36, + "cpu_freq": { + "current": 2400.032000000001, + "min": 0.0, + "max": 0.0 + }, + "cpu_freq_per_core": [ + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + }, + { + "current": 2400.032, + "min": 0.0, + "max": 0.0 + } + ], + "disk": { + "/": { + "total": 0.0625, + "used": 1.1444091796875e-05 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 2, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 113.18244934082031 + } +} diff --git a/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json b/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..51058c88c3a1625dd85005c8ee6a15896f91372c --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json @@ -0,0 +1 @@ +{"training/loss": 2.5753633975982666, "training/perplexity": 13.136089914759047, "utils/batch_size": 12, "utils/global_batch_size": 240, "utils/seq_len": 1025, "utils/gradient_accumulation_steps": 10, "utils/iteration": 9, "optimizer/lr": 3.235714285714285e-05, "optimizer/variance_l2": 0.007546109731502451, "optimizer/variance_sqrt_l2": 0.5729893739399086, "optimizer/momentum_l2": 0.19320277413391512, "optimizer/weight_l2": 778.0077120440388, "optimizer/variance_l1": 0.3284492492675781, "optimizer/variance_sqrt_l1": 5172.0, "optimizer/momentum_l1": 1788.5, "optimizer/weight_l1": 25657344.0, "optimizer/variance_abs_max": 0.005035400390625, "optimizer/variance_sqrt_abs_max": 0.07080078125, "optimizer/momentum_abs_max": 0.0145263671875, "optimizer/weight_abs_max": 6.5, "stats/1_iteration_time": 22.05749237000009, "stats/tokens_per_sec": 11152.673018016281, "stats/tokens_per_sec_per_gpu": 5576.336509008141, "stats/tflops": 122.2271178750492, "_timestamp": 1730400529.2158527, "_runtime": 215.62579369544983, "_step": 9, "_wandb": {"runtime": 215}} \ No newline at end of file diff --git a/wandb/run-20241101_034513-twy91j38/logs/debug-internal.log b/wandb/run-20241101_034513-twy91j38/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6d3df8a6f46a7f1af7761657be7129decd386667 --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/logs/debug-internal.log @@ -0,0 +1,292 @@ +2024-11-01 03:45:13,592 INFO StreamThr :14036 [internal.py:wandb_internal():86] W&B internal server running at pid: 14036, started at: 2024-11-01 03:45:13.591439 +2024-11-01 03:45:13,593 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status +2024-11-01 03:45:13,595 INFO WriterThread:14036 [datastore.py:open_for_write():87] open: /project/wandb/run-20241101_034513-twy91j38/run-twy91j38.wandb +2024-11-01 03:45:13,596 DEBUG SenderThread:14036 [sender.py:send():382] send: header +2024-11-01 03:45:13,611 DEBUG SenderThread:14036 [sender.py:send():382] send: run +2024-11-01 03:45:14,079 INFO SenderThread:14036 [dir_watcher.py:__init__():211] watching files in: /project/wandb/run-20241101_034513-twy91j38/files +2024-11-01 03:45:14,079 INFO SenderThread:14036 [sender.py:_start_run_threads():1136] run started: twy91j38 with start time 1730400313.590059 +2024-11-01 03:45:14,084 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: check_version +2024-11-01 03:45:14,084 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: check_version +2024-11-01 03:45:14,155 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: run_start +2024-11-01 03:45:14,162 DEBUG HandlerThread:14036 [system_info.py:__init__():27] System info init +2024-11-01 03:45:14,162 DEBUG HandlerThread:14036 [system_info.py:__init__():42] System info init done +2024-11-01 03:45:14,162 INFO HandlerThread:14036 [system_monitor.py:start():194] Starting system monitor +2024-11-01 03:45:14,162 INFO SystemMonitor:14036 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-11-01 03:45:14,162 INFO HandlerThread:14036 [system_monitor.py:probe():214] Collecting system info +2024-11-01 03:45:14,163 INFO SystemMonitor:14036 [interfaces.py:start():190] Started cpu monitoring +2024-11-01 03:45:14,163 INFO SystemMonitor:14036 [interfaces.py:start():190] Started disk monitoring +2024-11-01 03:45:14,164 INFO SystemMonitor:14036 [interfaces.py:start():190] Started gpu monitoring +2024-11-01 03:45:14,165 INFO SystemMonitor:14036 [interfaces.py:start():190] Started memory monitoring +2024-11-01 03:45:14,167 INFO SystemMonitor:14036 [interfaces.py:start():190] Started network monitoring +2024-11-01 03:45:14,181 DEBUG HandlerThread:14036 [system_info.py:probe():151] Probing system +2024-11-01 03:45:14,183 DEBUG HandlerThread:14036 [system_info.py:_probe_git():136] Probing git +2024-11-01 03:45:14,196 DEBUG HandlerThread:14036 [system_info.py:_probe_git():144] Probing git done +2024-11-01 03:45:14,197 DEBUG HandlerThread:14036 [system_info.py:probe():199] Probing system done +2024-11-01 03:45:14,197 DEBUG HandlerThread:14036 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-91-generic-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2024-10-31T18:45:14.181243', 'startedAt': '2024-10-31T18:45:13.576906', 'docker': None, 'cuda': None, 'args': ('--seq-length', '1024', '--sliding-window-size', '131072', '--micro-batch-size', '12', '--valid_micro_batch_size', '1', '--global-batch-size', '256', '--train-iters', '698', '--tokenizer-type', 'HFPreTrainedTokenizer', '--tokenizer-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--train-data-path', '183224836', '/project/trans/datasets/processed_ja-en/train/documents_text_document', '--valid-data-path', '61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document', '--test-data-path', '61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document', '--lr', '1.5e-4', '--min-lr', '1.5e-5', '--lr-decay-style', 'cosine', '--lr-warmup-iters', '70', '--lr-decay-iters', '698', '--weight-decay', '0.1', '--grad-clip-norm', '1.0', '--optimizer', 'anyprecision', '--adam-beta1', '0.9', '--adam-beta2', '0.95', '--adam-eps', '1e-8', '--save-interval', '50', '--eval-interval', '20', '--eval-iters', '10', '--bf16', '--mixed-precision', '--base-model', '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', '--save', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--load', '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--num-workers', '4', '--fsdp-activation-checkpointing', '--sharding-strategy', 'SHARD_GRAD_OP', '--checkpoint-type', 'LOCAL_STATE_DICT', '--save-n-checkpoints', '3', '--upload-all-checkpoints-to-hf', '--hf-upload-retry-limit', '2', '--hf-repo-id', 'koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', '--wandb-entity', 'iwakawa-koichi-q5-tohoku-nlp6723', '--wandb-project', 'trans_experiment', '--wandb-name', 'llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:44:59'), 'state': 'running', 'program': '/project/examples/finetuning.py', 'codePathLocal': 'examples/finetuning.py', 'codePath': 'examples/finetuning.py', 'git': {'remote': 'https://github.com/cl-tohoku/llm-recipes-failab-m1-yans.git', 'commit': '3b2976faebe2228c39adb20194a29b785a37defe'}, 'email': None, 'root': '/project', 'host': 'gpu-koiwa-00', 'username': 'koiwa', 'executable': '/usr/bin/python', 'cpu_count': 36, 'cpu_count_logical': 36, 'cpu_freq': {'current': 2400.032000000001, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}, {'current': 2400.032, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 0.0625, 'used': 1.1444091796875e-05}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 2, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 113.18244934082031}} +2024-11-01 03:45:14,197 INFO HandlerThread:14036 [system_monitor.py:probe():224] Finished collecting system info +2024-11-01 03:45:14,197 INFO HandlerThread:14036 [system_monitor.py:probe():227] Publishing system info +2024-11-01 03:45:14,198 INFO HandlerThread:14036 [system_monitor.py:probe():229] Finished publishing system info +2024-11-01 03:45:14,204 DEBUG SenderThread:14036 [sender.py:send():382] send: files +2024-11-01 03:45:14,204 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-11-01 03:45:14,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: python_packages +2024-11-01 03:45:14,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:45:14,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:14,216 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: python_packages +2024-11-01 03:45:14,218 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:45:14,409 DEBUG SenderThread:14036 [sender.py:send():382] send: telemetry +2024-11-01 03:45:14,878 INFO wandb-upload_0:14036 [upload_job.py:push():131] Uploaded file /tmp/tmpqveipv3mwandb/jpj7fuam-wandb-metadata.json +2024-11-01 03:45:15,081 INFO Thread-12 :14036 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_034513-twy91j38/files/requirements.txt +2024-11-01 03:45:15,081 INFO Thread-12 :14036 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_034513-twy91j38/files/wandb-metadata.json +2024-11-01 03:45:15,081 INFO Thread-12 :14036 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:17,082 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:18,949 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:21,084 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:24,723 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:25,087 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:28,319 DEBUG SenderThread:14036 [sender.py:send():382] send: config +2024-11-01 03:45:28,319 DEBUG SenderThread:14036 [sender.py:send():382] send: config +2024-11-01 03:45:29,090 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:29,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:29,218 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:45:29,218 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:45:30,402 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:31,091 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:35,403 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:40,403 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:44,215 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:45:44,216 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:45:44,258 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:45,422 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:46,101 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/config.yaml +2024-11-01 03:45:50,623 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:52,190 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:45:55,107 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:45:56,232 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:45:59,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:45:59,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:45:59,216 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:46:01,395 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:06,396 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:11,397 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:14,167 DEBUG SystemMonitor:14036 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-11-01 03:46:14,170 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:46:14,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:46:14,216 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:46:14,258 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:14,814 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:46:14,817 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:46:14,817 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:46:14,819 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:46:15,120 INFO Thread-12 :14036 [dir_watcher.py:_on_file_created():271] file/dir created: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:46:16,856 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:17,121 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:46:21,856 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:26,857 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:29,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:46:29,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:29,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:46:32,465 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:36,873 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:46:36,875 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:46:36,876 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:46:36,878 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:46:37,135 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:46:37,916 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:39,136 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:46:42,916 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:44,171 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:46:44,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:46:44,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:44,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:46:48,469 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:53,469 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:58,470 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:46:58,938 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:46:58,940 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:46:58,940 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:46:58,941 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:46:59,149 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:46:59,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:46:59,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:46:59,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:47:01,150 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:47:03,475 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:08,476 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:13,476 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:14,172 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:47:14,216 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:47:14,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:47:14,258 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:19,425 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:20,994 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:47:20,996 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:47:20,997 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:47:20,999 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:47:21,163 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:47:23,165 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:47:25,036 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:29,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:47:29,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:29,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:47:30,421 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:35,421 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:40,422 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:43,058 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:47:43,060 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:47:43,060 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:47:43,062 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:47:43,179 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:47:44,176 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:47:44,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:47:44,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:47:44,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:47:45,180 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:47:45,470 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:50,471 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:55,472 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:47:59,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:47:59,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:47:59,258 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:01,431 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:05,117 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:48:05,119 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:48:05,120 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:48:05,122 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:05,193 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:48:07,160 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:07,194 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:48:12,160 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:14,174 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:48:14,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:48:14,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:48:14,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:17,431 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:22,431 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:27,157 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:48:27,159 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:48:27,160 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:48:27,161 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:27,208 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:48:28,200 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:29,209 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:48:29,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:48:29,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:48:29,219 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:33,435 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:38,436 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:43,436 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:44,176 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:48:44,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: stop_status +2024-11-01 03:48:44,217 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: stop_status +2024-11-01 03:48:44,258 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: internal_messages +2024-11-01 03:48:48,500 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:49,217 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: partial_history +2024-11-01 03:48:49,219 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:48:49,219 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:48:49,221 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:49,223 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json +2024-11-01 03:48:49,302 DEBUG SenderThread:14036 [sender.py:send():382] send: exit +2024-11-01 03:48:49,302 INFO SenderThread:14036 [sender.py:send_exit():589] handling exit code: 1 +2024-11-01 03:48:49,302 INFO SenderThread:14036 [sender.py:send_exit():591] handling runtime: 215 +2024-11-01 03:48:49,303 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:49,303 INFO SenderThread:14036 [sender.py:send_exit():597] send defer +2024-11-01 03:48:49,303 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,304 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 0 +2024-11-01 03:48:49,304 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,304 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 0 +2024-11-01 03:48:49,304 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 1 +2024-11-01 03:48:49,304 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,304 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 1 +2024-11-01 03:48:49,304 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,304 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 1 +2024-11-01 03:48:49,304 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 2 +2024-11-01 03:48:49,304 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,304 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 2 +2024-11-01 03:48:49,304 INFO HandlerThread:14036 [system_monitor.py:finish():203] Stopping system monitor +2024-11-01 03:48:49,305 DEBUG SystemMonitor:14036 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-11-01 03:48:49,305 INFO HandlerThread:14036 [interfaces.py:finish():202] Joined cpu monitor +2024-11-01 03:48:49,305 DEBUG SystemMonitor:14036 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-11-01 03:48:49,305 INFO HandlerThread:14036 [interfaces.py:finish():202] Joined disk monitor +2024-11-01 03:48:49,408 INFO HandlerThread:14036 [interfaces.py:finish():202] Joined gpu monitor +2024-11-01 03:48:49,408 INFO HandlerThread:14036 [interfaces.py:finish():202] Joined memory monitor +2024-11-01 03:48:49,408 INFO HandlerThread:14036 [interfaces.py:finish():202] Joined network monitor +2024-11-01 03:48:49,409 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,409 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 2 +2024-11-01 03:48:49,409 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 3 +2024-11-01 03:48:49,409 DEBUG SenderThread:14036 [sender.py:send():382] send: stats +2024-11-01 03:48:49,409 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,410 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 3 +2024-11-01 03:48:49,411 DEBUG SenderThread:14036 [sender.py:send():382] send: history +2024-11-01 03:48:49,411 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: summary_record +2024-11-01 03:48:49,412 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:49,413 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,413 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 3 +2024-11-01 03:48:49,413 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 4 +2024-11-01 03:48:49,413 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,413 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 4 +2024-11-01 03:48:49,413 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,413 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 4 +2024-11-01 03:48:49,413 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 5 +2024-11-01 03:48:49,413 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,413 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 5 +2024-11-01 03:48:49,414 DEBUG SenderThread:14036 [sender.py:send():382] send: summary +2024-11-01 03:48:49,415 INFO SenderThread:14036 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-11-01 03:48:49,415 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,415 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 5 +2024-11-01 03:48:49,415 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 6 +2024-11-01 03:48:49,415 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,415 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 6 +2024-11-01 03:48:49,416 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,416 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 6 +2024-11-01 03:48:49,416 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 7 +2024-11-01 03:48:49,416 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:48:49,416 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:49,416 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 7 +2024-11-01 03:48:49,416 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:49,416 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 7 +2024-11-01 03:48:50,302 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 03:48:51,159 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 8 +2024-11-01 03:48:51,159 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 03:48:51,159 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:51,160 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 8 +2024-11-01 03:48:51,160 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:51,160 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 8 +2024-11-01 03:48:51,160 INFO SenderThread:14036 [job_builder.py:build():296] Attempting to build job artifact +2024-11-01 03:48:51,161 INFO SenderThread:14036 [job_builder.py:_get_source_type():426] is repo sourced job +2024-11-01 03:48:51,176 INFO SenderThread:14036 [job_builder.py:build():402] adding wandb-job metadata file +2024-11-01 03:48:51,185 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 9 +2024-11-01 03:48:51,186 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:51,186 DEBUG SenderThread:14036 [sender.py:send():382] send: artifact +2024-11-01 03:48:51,186 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 9 +2024-11-01 03:48:51,225 INFO Thread-12 :14036 [dir_watcher.py:_on_file_modified():288] file/dir modified: /project/wandb/run-20241101_034513-twy91j38/files/output.log +2024-11-01 03:48:51,302 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: poll_exit +2024-11-01 03:48:52,049 INFO SenderThread:14036 [sender.py:send_artifact():1494] sent artifact job-https___github.com_cl-tohoku_llm-recipes-failab-m1-yans.git_examples_finetuning.py - {'id': 'QXJ0aWZhY3Q6MTMwMjQ5MDY3NQ==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjQ4MzQ0MjQxNg==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTMwMjQ5MDY3NQ==', 'versionIndex': 1}}} +2024-11-01 03:48:52,049 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:52,049 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 9 +2024-11-01 03:48:52,049 INFO SenderThread:14036 [dir_watcher.py:finish():358] shutting down directory watcher +2024-11-01 03:48:52,225 INFO SenderThread:14036 [dir_watcher.py:finish():388] scan: /project/wandb/run-20241101_034513-twy91j38/files +2024-11-01 03:48:52,226 INFO SenderThread:14036 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_034513-twy91j38/files/requirements.txt requirements.txt +2024-11-01 03:48:52,226 INFO SenderThread:14036 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_034513-twy91j38/files/config.yaml config.yaml +2024-11-01 03:48:52,226 INFO SenderThread:14036 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_034513-twy91j38/files/wandb-metadata.json wandb-metadata.json +2024-11-01 03:48:52,226 INFO SenderThread:14036 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_034513-twy91j38/files/wandb-summary.json wandb-summary.json +2024-11-01 03:48:52,227 INFO SenderThread:14036 [dir_watcher.py:finish():402] scan save: /project/wandb/run-20241101_034513-twy91j38/files/output.log output.log +2024-11-01 03:48:52,227 INFO SenderThread:14036 [sender.py:transition_state():617] send defer: 10 +2024-11-01 03:48:52,227 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: poll_exit +2024-11-01 03:48:52,227 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: defer +2024-11-01 03:48:52,227 INFO HandlerThread:14036 [handler.py:handle_request_defer():172] handle defer: 10 +2024-11-01 03:48:52,227 DEBUG SenderThread:14036 [sender.py:send_request():409] send_request: defer +2024-11-01 03:48:52,227 INFO SenderThread:14036 [sender.py:send_request_defer():613] handle sender defer: 10 +2024-11-01 03:48:52,227 INFO SenderThread:14036 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 03:48:59,228 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:04,229 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:09,230 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:14,230 DEBUG HandlerThread:14036 [handler.py:handle_request():146] handle_request: status_report +2024-11-01 03:49:14,837 WARNING StreamThr :14036 [internal.py:is_dead():414] Internal process exiting, parent pid 13869 disappeared +2024-11-01 03:49:14,838 ERROR StreamThr :14036 [internal.py:wandb_internal():152] Internal process shutdown. +2024-11-01 03:49:15,230 INFO SenderThread:14036 [sender.py:finish():1572] shutting down sender +2024-11-01 03:49:15,230 INFO SenderThread:14036 [file_pusher.py:finish():172] shutting down file pusher +2024-11-01 03:49:15,230 INFO SenderThread:14036 [file_pusher.py:join():178] waiting for file pusher +2024-11-01 03:49:15,230 INFO HandlerThread:14036 [handler.py:finish():869] shutting down handler +2024-11-01 03:49:15,231 INFO WriterThread:14036 [datastore.py:close():296] close: /project/wandb/run-20241101_034513-twy91j38/run-twy91j38.wandb +2024-11-01 03:49:15,231 INFO SenderThread:14036 [file_stream.py:finish():595] file stream finish called +2024-11-01 03:49:15,519 INFO SenderThread:14036 [file_stream.py:finish():599] file stream finish is done diff --git a/wandb/run-20241101_034513-twy91j38/logs/debug.log b/wandb/run-20241101_034513-twy91j38/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3e3ddc4876bc9fb9b3b59b76db22e161b6dc8c12 --- /dev/null +++ b/wandb/run-20241101_034513-twy91j38/logs/debug.log @@ -0,0 +1,29 @@ +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Current SDK version is 0.16.3 +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Configure stats pid to 13869 +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Loading settings from /singularity_home/.config/wandb/settings +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Loading settings from /project/wandb/settings +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'api_key': '***REDACTED***', 'run_notes': 'CPT en-ja'} +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'examples/finetuning.py', 'program_abspath': '/project/examples/finetuning.py', 'program': '/project/examples/finetuning.py'} +2024-11-01 03:45:13,583 INFO MainThread:13869 [wandb_init.py:_log_setup():526] Logging user logs to /project/wandb/run-20241101_034513-twy91j38/logs/debug.log +2024-11-01 03:45:13,584 INFO MainThread:13869 [wandb_init.py:_log_setup():527] Logging internal logs to /project/wandb/run-20241101_034513-twy91j38/logs/debug-internal.log +2024-11-01 03:45:13,584 INFO MainThread:13869 [wandb_init.py:init():566] calling init triggers +2024-11-01 03:45:13,584 INFO MainThread:13869 [wandb_init.py:init():573] wandb.init called with sweep_config: {} +config: {'sharding_strategy': 'SHARD_GRAD_OP', 'checkpoint_type': 'LOCAL_STATE_DICT', 'fsdp_activation_checkpointing': True, 'fsdp_cpu_offload': False, 'low_cpu_fsdp': False, 'no_meta_device': False, 'data_path': None, 'split': '969, 30, 1', 'train_data_path': ['183224836', '/project/trans/datasets/processed_ja-en/train/documents_text_document'], 'valid_data_path': ['61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document'], 'test_data_path': ['61084836', '/project/trans/datasets/processed_ja-en/valid/documents_text_document'], 'data_cache_path': None, 'vocab_size': None, 'vocab_file': None, 'merge_file': None, 'seq_length': 1024, 'num_workers': 4, 'tokenizer_type': 'HFPreTrainedTokenizer', 'tokenizer_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'reset_position_ids': False, 'reset_attention_mask': False, 'eod_mask_loss': False, 'retro_return_doc_ids': False, 'short_seq_prob': 0.1, 'vocab_extra_ids': 0, 'seed': 1234, 'use_mpi': False, 'wandb_entity': 'iwakawa-koichi-q5-tohoku-nlp6723', 'wandb_name': 'llm-jp-v3-3.7b_ja-en_actual_3M-pairs_train_2024-11-01-03:44:59', 'wandb_project': 'trans_experiment', 'quantization': False, 'use_freeze_layers': False, 'freeze_layers': None, 'bf16': True, 'fp16': False, 'mixed_precision': True, 'param_dtype': None, 'load': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'save': '/work/llm_recipes/models/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'base_model': '/share/pretrained_lm/llm-jp/llm-jp-v3-3.7b', 'use_better_transformer': False, 'grad_clip_norm': 1.0, 'eval_interval': 20, 'save_interval': 50, 'eval_iters': 10, 'optimizer': 'anyprecision', 'lr': 0.00015, 'lr_decay_style': 'cosine', 'lr_decay_iters': 698, 'lr_warmup_iters': 70, 'min_lr': 1.5e-05, 'train_iters': 698, 'train_samples': None, 'global_batch_size': 256, 'micro_batch_size': 12, 'make_vocab_size_divisible_by': 128, 'sliding_window_size': 131072, 'skip_batch': None, 'no_save_optimizer_state': False, 'continual_pretraining': False, 'instruction_tuning': False, 'direct_preference_optimization': False, 'attention_dropout': 0.1, 'hidden_dropout': 0.1, 'weight_decay': 0.1, 'adam_beta1': 0.9, 'adam_beta2': 0.95, 'adam_eps': 1e-08, 'hf_transformer_model_dir': None, 'instruction_train_data_path': None, 'instruction_valid_data_path': None, 'epoch': None, 'instruction_dataset_size': None, 'save_sampler_state': False, 'label_smoothing': 0.0, 'save_n_checkpoints': 3, 'hf_repo_id': 'koichi12/llm-jp-v3-3.7b_ja-en_actual_3M-pairs', 'create_public_hf_repo': False, 'upload_all_checkpoints_to_hf': True, 'hf_upload_retry_limit': 2, 'exit_duration_in_mins': None, 'source_key': None, 'target_key': None, 'attn_implementation': 'flash_attention_2', 'efficient_instruction_tuning': False, 'remove_padding_masking': False, 'save_start_iter': None, 'valid_micro_batch_size': 1, 'rank': 0, 'world_size': 2, 'padded_vocab_size': 99584, 'gradient_accumulation_steps': 10} +2024-11-01 03:45:13,584 INFO MainThread:13869 [wandb_init.py:init():616] starting backend +2024-11-01 03:45:13,584 INFO MainThread:13869 [wandb_init.py:init():620] setting up manager +2024-11-01 03:45:13,589 INFO MainThread:13869 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-11-01 03:45:13,589 INFO MainThread:13869 [wandb_init.py:init():628] backend started and connected +2024-11-01 03:45:13,595 INFO MainThread:13869 [wandb_init.py:init():720] updated telemetry +2024-11-01 03:45:13,607 INFO MainThread:13869 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2024-11-01 03:45:14,083 INFO MainThread:13869 [wandb_run.py:_on_init():2262] communicating current version +2024-11-01 03:45:14,108 INFO MainThread:13869 [wandb_run.py:_on_init():2271] got version response upgrade_message: "wandb version 0.18.5 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2024-11-01 03:45:14,108 INFO MainThread:13869 [wandb_init.py:init():804] starting run threads in backend +2024-11-01 03:45:14,214 INFO MainThread:13869 [wandb_run.py:_console_start():2241] atexit reg +2024-11-01 03:45:14,214 INFO MainThread:13869 [wandb_run.py:_redirect():2096] redirect: wrap_raw +2024-11-01 03:45:14,215 INFO MainThread:13869 [wandb_run.py:_redirect():2161] Wrapping output streams. +2024-11-01 03:45:14,215 INFO MainThread:13869 [wandb_run.py:_redirect():2186] Redirects installed. +2024-11-01 03:45:14,216 INFO MainThread:13869 [wandb_init.py:init():847] run started, returning control to user process +2024-11-01 03:45:28,318 INFO MainThread:13869 [wandb_run.py:_config_callback():1343] config_cb None None {'model_architecture': 'LlamaForCausalLM', 'activation_function': 'silu', 'hidden_size': 3072, 'model_type': 'llama', 'max_position_embeddings': 4096, 'num_attention_heads': 24, 'num_hidden_layers': 28} +2024-11-01 03:45:28,319 INFO MainThread:13869 [wandb_run.py:_config_callback():1343] config_cb None None {'world_size': 2} diff --git a/wandb/run-20241101_034513-twy91j38/run-twy91j38.wandb b/wandb/run-20241101_034513-twy91j38/run-twy91j38.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ce23bfed5b601f08aaa0a8eb51b1acd192a3f49c Binary files /dev/null and b/wandb/run-20241101_034513-twy91j38/run-twy91j38.wandb differ